From f749d4e00b4b87b76902fd4e8a837fd50398e827 Mon Sep 17 00:00:00 2001 From: Joey Lee Date: Thu, 12 Dec 2019 11:57:11 -0500 Subject: [PATCH] [nn-refactor] Refactor and reimplementation of NeuralNetwork (#749) * adds temp DiyNeuralNetwork - refactoring NeuralNetwork class * adds updates to refactor * refactoring nn-refactor * adds features for compile and add layers * rm console log * adds train interface * adds basic predict * adds blank functions in data class * update nn class * adds nn compile handling * updates function name * adds data loading functions // todo - clean up * add recursive findEntries function and data loading functions * adds formatRawData function * adds .addData() function * adds saveData function * adds handling for onehot and counting input and output units " " * adds code comments * adds concat to this.meta * changed name to createMetaDataFromData" * adds convertRawToTensors * adds functions for calculating stats * adds normalization and conversion to tensor handling * adds .summarizeData * adds data handling to index * updates summarizeData function to explicitly set meta * updates and adds functions * updates predict function * adds classify() with meta * adds metadata handling and data functions * adds loadData with options in init * adds major updates to initiation and defaults * adds boolean flags to check status to configure nn * adds addData function to index * adds support for auto labeling inputs and outputs for blank nn * code cleanup and function name change * flattens array in cnvertRawToTensors * flattens inputs * flatten array always * adds isOneHotEncodedOrNormalized * updates predict and classify functions and output format * updates param handling in predict and classify * code cleanup * adds save function * code cleanup * adds first pass at loading data * fixes missing isNormalized flag in meta * moves loading functions to respective class * moves files to NeuralNetwork * moves files to NeuralNetwork and rm diyNN * rms console.log * check if metadata and warmedup are true before normalization * adds unnormalize function to nn predict * return unNormalized value * adds loadData() and changes to loadDataFromUrl * adds saveData to index * adds modelUrl to constructor options in index * cleans up predict and classify * fix reference to unNormalizeValue * code cleanup * adds looping to format data for prediction and predictMultiple and classifyMultiple * adds layer handling for options * adds tfvis to index and ml5 root * adds debug flag in options * adds vis and fixes input formatting" " * adds model summary * adds comments and reorders code * refactoring functions with 3 datatypes in mind: number, string, array * adds data handling updates * adds handling tensors * adds process up to training * fixes breaking training * adds full working poc * fix addData check * adds updates to api and notes to fix with functions * adds createMetadata in index * adds image handling in classify functino * adds method to not exceed call stack of min and max * fixes loadData issue * adds first header name for min and max * code cleanup * removes unused functions * fixes setDataRaw * code clean up, organization, and adds method binding to constructor * adds methods to constructor, adds comments, and cleans up * adds methods to constructor for nndata * adds methods to constructor, code cleanup, and organization --- src/NeuralNetwork/NeuralNetwork.js | 249 ++++ src/NeuralNetwork/NeuralNetworkData.js | 1306 ++++++++++------- src/NeuralNetwork/NeuralNetworkDefaults.js | 19 - src/NeuralNetwork/NeuralNetworkUtils.js | 151 ++ src/NeuralNetwork/NeuralNetworkVis.js | 9 +- src/NeuralNetwork/index.js | 1529 ++++++++++---------- src/NeuralNetwork/index_test.js | 63 - src/index.js | 2 + src/utils/imageUtilities.js | 48 +- 9 files changed, 2053 insertions(+), 1323 deletions(-) create mode 100644 src/NeuralNetwork/NeuralNetwork.js delete mode 100644 src/NeuralNetwork/NeuralNetworkDefaults.js create mode 100644 src/NeuralNetwork/NeuralNetworkUtils.js diff --git a/src/NeuralNetwork/NeuralNetwork.js b/src/NeuralNetwork/NeuralNetwork.js new file mode 100644 index 000000000..41b28449c --- /dev/null +++ b/src/NeuralNetwork/NeuralNetwork.js @@ -0,0 +1,249 @@ +import * as tf from '@tensorflow/tfjs'; +import callCallback from '../utils/callcallback'; +import { saveBlob } from '../utils/io'; + +class NeuralNetwork { + constructor() { + // flags + this.isTrained = false; + this.isCompiled = false; + this.isLayered = false; + // the model + this.model = null; + + // methods + this.init = this.init.bind(this); + this.createModel = this.createModel.bind(this); + this.addLayer = this.addLayer.bind(this); + this.compile = this.compile.bind(this); + this.setOptimizerFunction = this.setOptimizerFunction.bind(this); + this.train = this.train.bind(this); + this.trainInternal = this.trainInternal.bind(this); + this.predict = this.predict.bind(this); + this.classify = this.classify.bind(this); + this.save = this.save.bind(this); + this.load = this.load.bind(this); + + // initialize + this.init(); + } + + /** + * initialize with create model + */ + init() { + this.createModel(); + } + + /** + * creates a sequential model + * uses switch/case for potential future where different formats are supported + * @param {*} _type + */ + createModel(_type = 'sequential') { + switch (_type.toLowerCase()) { + case 'sequential': + this.model = tf.sequential(); + return this.model; + default: + this.model = tf.sequential(); + return this.model; + } + } + + /** + * add layer to the model + * if the model has 2 or more layers switch the isLayered flag + * @param {*} _layerOptions + */ + addLayer(_layerOptions) { + const LAYER_OPTIONS = _layerOptions || {}; + this.model.add(LAYER_OPTIONS); + + // check if it has at least an input and output layer + if (this.model.layers.length >= 2) { + this.isLayered = true; + } + } + + /** + * Compile the model + * if the model is compiled, set the isCompiled flag to true + * @param {*} _modelOptions + */ + compile(_modelOptions) { + this.model.compile(_modelOptions); + this.isCompiled = true; + } + + /** + * Set the optimizer function given the learning rate + * as a paramter + * @param {*} learningRate + * @param {*} optimizer + */ + setOptimizerFunction(learningRate, optimizer) { + return optimizer.call(this, learningRate); + } + + /** + * Calls the trainInternal() and calls the callback when finished + * @param {*} _options + * @param {*} _cb + */ + train(_options, _cb) { + return callCallback(this.trainInternal(_options), _cb); + } + + /** + * Train the model + * @param {*} _options + */ + async trainInternal(_options) { + const TRAINING_OPTIONS = _options; + + const xs = TRAINING_OPTIONS.inputs; + const ys = TRAINING_OPTIONS.outputs; + + const { batchSize, epochs, shuffle, validationSplit, whileTraining } = TRAINING_OPTIONS; + + await this.model.fit(xs, ys, { + batchSize, + epochs, + shuffle, + validationSplit, + callbacks: whileTraining, + }); + + xs.dispose(); + ys.dispose(); + + this.isTrained = true; + } + + /** + * returns the prediction as an array + * @param {*} _inputs + */ + async predict(_inputs) { + const output = tf.tidy(() => { + return this.model.predict(_inputs); + }); + const result = await output.array(); + + output.dispose(); + _inputs.dispose(); + + return result; + } + + /** + * classify is the same as .predict() + * @param {*} _inputs + */ + async classify(_inputs) { + return this.predict(_inputs); + } + + // predictMultiple + // classifyMultiple + // are the same as .predict() + + /** + * save the model + * @param {*} nameOrCb + * @param {*} cb + */ + async save(nameOrCb, cb) { + let modelName; + let callback; + + if (typeof nameOrCb === 'function') { + modelName = 'model'; + callback = nameOrCb; + } else if (typeof nameOrCb === 'string') { + modelName = nameOrCb; + + if (typeof cb === 'function') { + callback = cb; + } + } else { + modelName = 'model'; + } + + this.model.save( + tf.io.withSaveHandler(async data => { + this.weightsManifest = { + modelTopology: data.modelTopology, + weightsManifest: [ + { + paths: [`./${modelName}.weights.bin`], + weights: data.weightSpecs, + }, + ], + }; + + await saveBlob(data.weightData, `${modelName}.weights.bin`, 'application/octet-stream'); + await saveBlob(JSON.stringify(this.weightsManifest), `${modelName}.json`, 'text/plain'); + if (callback) { + callback(); + } + }), + ); + } + + /** + * loads the model and weights + * @param {*} filesOrPath + * @param {*} callback + */ + async load(filesOrPath = null, callback) { + if (filesOrPath instanceof FileList) { + const files = await Promise.all( + Array.from(filesOrPath).map(async file => { + if (file.name.includes('.json') && !file.name.includes('_meta')) { + return { name: 'model', file }; + } else if (file.name.includes('.json') && file.name.includes('_meta.json')) { + const modelMetadata = await file.text(); + return { name: 'metadata', file: modelMetadata }; + } else if (file.name.includes('.bin')) { + return { name: 'weights', file }; + } + return { name: null, file: null }; + }), + ); + + const model = files.find(item => item.name === 'model').file; + const weights = files.find(item => item.name === 'weights').file; + + // load the model + this.model = await tf.loadLayersModel(tf.io.browserFiles([model, weights])); + } else if (filesOrPath instanceof Object) { + // filesOrPath = {model: URL, metadata: URL, weights: URL} + + let modelJson = await fetch(filesOrPath.model); + modelJson = await modelJson.text(); + const modelJsonFile = new File([modelJson], 'model.json', { type: 'application/json' }); + + let weightsBlob = await fetch(filesOrPath.weights); + weightsBlob = await weightsBlob.blob(); + const weightsBlobFile = new File([weightsBlob], 'model.weights.bin', { + type: 'application/macbinary', + }); + + this.model = await tf.loadLayersModel(tf.io.browserFiles([modelJsonFile, weightsBlobFile])); + } else { + this.model = await tf.loadLayersModel(filesOrPath); + } + + this.isCompiled = true; + this.isLayered = true; + this.isTrained = true; + + if (callback) { + callback(); + } + return this.model; + } +} +export default NeuralNetwork; diff --git a/src/NeuralNetwork/NeuralNetworkData.js b/src/NeuralNetwork/NeuralNetworkData.js index 3924e84e2..aa368d7fc 100644 --- a/src/NeuralNetwork/NeuralNetworkData.js +++ b/src/NeuralNetwork/NeuralNetworkData.js @@ -1,633 +1,961 @@ import * as tf from '@tensorflow/tfjs'; -import { - saveBlob -} from '../utils/io'; -// import * as tfvis from '@tensorflow/tfjs-vis'; -// import callCallback from '../utils/callcallback'; +import { saveBlob } from '../utils/io'; +import nnUtils from './NeuralNetworkUtils'; -/* eslint class-methods-use-this: ["error", { "exceptMethods": ["shuffle", "normalizeArray"] }] */ class NeuralNetworkData { - constructor(options) { - this.config = options; + constructor() { this.meta = { - // number of units - varies depending on input data type - inputUnits: null, - outputUnits: null, + inputUnits: null, // Number + outputUnits: null, // Number // objects describing input/output data by property name inputs: {}, // { name1: {dtype}, name2: {dtype} } outputs: {}, // { name1: {dtype} } - isNormalized: false, - } + isNormalized: false, // Boolean - keep this in meta for model saving/loading + }; - this.data = { - raw: [], - tensor: { - inputs: null, // tensor - outputs: null, // tensor - inputMax: null, // tensor - inputMin: null, // tensor - outputMax: null, // tensor - outputMin: null, // tensor - }, - inputMax: null, // array or number - inputMin: null, // array or number - outputMax: null, // array or number - outputMin: null, // array or number - } - - // TODO: temp fix - check normalizationOptions - if (options.dataOptions.normalizationOptions !== null && options.dataOptions.normalizationOptions !== undefined) { - const items = ['inputMax', 'inputMin', 'outputMax', 'outputMin']; - items.forEach(prop => { - if (options.dataOptions.normalizationOptions[prop] !== null && options.dataOptions.normalizationOptions[prop] !== undefined) { - this.data[prop] = options.dataOptions.normalizationOptions[prop] - } - }) - } + this.isMetadataReady = false; + this.isWarmedUp = false; + this.data = { + raw: [], // array of {xs:{}, ys:{}} + }; + + // methods + // summarize data + this.createMetadata = this.createMetadata.bind(this); + this.getDataStats = this.getDataStats.bind(this); + this.getInputMetaStats = this.getInputMetaStats.bind(this); + this.getDataUnits = this.getDataUnits.bind(this); + this.getInputMetaUnits = this.getInputMetaUnits.bind(this); + this.getDTypesFromData = this.getDTypesFromData.bind(this); + // add data + this.addData = this.addData.bind(this); + // data conversion + this.convertRawToTensors = this.convertRawToTensors.bind(this); + // data normalization / unnormalization + this.normalizeDataRaw = this.normalizeDataRaw.bind(this); + this.normalizeInputData = this.normalizeInputData.bind(this); + this.normalizeArray = this.normalizeArray.bind(this); + this.unnormalizeArray = this.unnormalizeArray.bind(this); + // one hot + this.applyOneHotEncodingsToDataRaw = this.applyOneHotEncodingsToDataRaw.bind(this); + this.getDataOneHot = this.getDataOneHot.bind(this); + this.getInputMetaOneHot = this.getInputMetaOneHot.bind(this); + this.createOneHotEncodings = this.createOneHotEncodings.bind(this); + // Saving / loading data + this.loadDataFromUrl = this.loadDataFromUrl.bind(this); + this.loadJSON = this.loadJSON.bind(this); + this.loadCSV = this.loadCSV.bind(this); + this.loadBlob = this.loadBlob.bind(this); + this.loadData = this.loadData.bind(this); + this.saveData = this.saveData.bind(this); + this.saveMeta = this.saveMeta.bind(this); + this.loadMeta = this.loadMeta.bind(this); + // data loading helpers + this.findEntries = this.findEntries.bind(this); + this.formatRawData = this.formatRawData.bind(this); + this.csvToJSON = this.csvToJSON.bind(this); } - - + /** + * //////////////////////////////////////////////////////// + * Summarize Data + * //////////////////////////////////////////////////////// + */ /** - * load data + * create the metadata from the data + * this covers: + * 1. getting the datatype from the data + * 2. getting the min and max from the data + * 3. getting the oneHot encoded values + * 4. getting the inputShape and outputUnits from the data + * @param {*} dataRaw + * @param {*} inputShape */ - async loadData() { - const { - dataUrl - } = this.config.dataOptions; - if (dataUrl.endsWith('.csv')) { - await this.loadCSVInternal(); - } else if (dataUrl.endsWith('.json')) { - await this.loadJSONInternal(); - } else if (dataUrl.includes('blob')) { - await this.loadBlobInternal() - } else { - console.log('Not a valid data format. Must be csv or json') - } + createMetadata(dataRaw, inputShape = null) { + // get the data type for each property + this.getDTypesFromData(dataRaw); + // get the stats - min, max + this.getDataStats(dataRaw); + // onehot encode + this.getDataOneHot(dataRaw); + // calculate the input units from the data + this.getDataUnits(dataRaw, inputShape); + + this.isMetadataReady = true; + return { ...this.meta }; } + /* + * //////////////////////////////////////////////// + * data Summary + * //////////////////////////////////////////////// + */ + /** - * load csv - * TODO: pass to loadJSONInternal() + * get stats about the data + * @param {*} dataRaw */ - async loadCSVInternal() { - const path = this.config.dataOptions.dataUrl; - const myCsv = tf.data.csv(path); - const loadedData = await myCsv.toArray(); - const json = { - entries: loadedData - } - this.loadJSONInternal(json); + getDataStats(dataRaw) { + const meta = Object.assign({}, this.meta); + + const inputMeta = this.getInputMetaStats(dataRaw, meta.inputs, 'xs'); + const outputMeta = this.getInputMetaStats(dataRaw, meta.outputs, 'ys'); + + meta.inputs = inputMeta; + meta.outputs = outputMeta; + + this.meta = { + ...this.meta, + ...meta, + }; + + return meta; } /** - * load json data - * @param {*} parsedJson + * getRawStats + * get back the min and max of each label + * @param {*} dataRaw + * @param {*} inputOrOutputMeta + * @param {*} xsOrYs */ - async loadJSONInternal(parsedJson) { - const { - dataUrl - } = this.config.dataOptions; - const outputLabels = this.config.dataOptions.outputs; - const inputLabels = this.config.dataOptions.inputs; + // eslint-disable-next-line no-unused-vars, class-methods-use-this + getInputMetaStats(dataRaw, inputOrOutputMeta, xsOrYs) { + const inputMeta = Object.assign({}, inputOrOutputMeta); + + Object.keys(inputMeta).forEach(k => { + if (inputMeta[k].dtype === 'string') { + inputMeta[k].min = 0; + inputMeta[k].max = 1; + } else if (inputMeta[k].dtype === 'number') { + const dataAsArray = dataRaw.map(item => item[xsOrYs][k]); + inputMeta[k].min = nnUtils.getMin(dataAsArray); + inputMeta[k].max = nnUtils.getMax(dataAsArray); + } else if (inputMeta[k].dtype === 'array') { + const dataAsArray = dataRaw.map(item => item[xsOrYs][k]).flat(); + inputMeta[k].min = nnUtils.getMin(dataAsArray); + inputMeta[k].max = nnUtils.getMax(dataAsArray); + } + }); - let json; - // handle loading parsedJson - if (parsedJson instanceof Object) { - json = parsedJson; - } else { - const data = await fetch(dataUrl); - json = await data.json(); - } + return inputMeta; + } - // TODO: recurse through the object to find - // which object contains the - let parentProp; - if (Object.keys(json).includes('entries')) { - parentProp = 'entries' - } else if (Object.keys(json).includes('data')) { - parentProp = 'data' + /** + * get the data units, inputshape and output units + * @param {*} dataRaw + */ + getDataUnits(dataRaw, _arrayShape = null) { + const arrayShape = _arrayShape !== null ? _arrayShape : undefined; + const meta = Object.assign({}, this.meta); + + // if the data has a shape pass it in + let inputShape; + if (arrayShape) { + inputShape = arrayShape; } else { - console.log(`your data must be contained in an array in \n - a property called 'entries' or 'data'`); - return; + inputShape = [this.getInputMetaUnits(dataRaw, meta.inputs)].flat(); } - const dataArray = json[parentProp]; - - this.data.raw = dataArray.map((item) => { - - const output = { - xs: {}, - ys: {} - } - // TODO: keep an eye on the order of the - // property name order if you use the order - // later on in the code! - const props = Object.keys(item); - - props.forEach(prop => { - if (inputLabels.includes(prop)) { - output.xs[prop] = item[prop] - } + console.log(inputShape); - if (outputLabels.includes(prop)) { - output.ys[prop] = item[prop] - // convert ys into strings, if the task is classification - if (this.config.architecture.task === "classification" && typeof output.ys[prop] !== "string") { - output.ys[prop] += ""; - } - } - }) + const outputShape = this.getInputMetaUnits(dataRaw, meta.outputs); - return output; - }) + meta.inputUnits = inputShape; + meta.outputUnits = outputShape; - // set the data types for the inputs and outputs - this.setDTypes(); + this.meta = { + ...this.meta, + ...meta, + }; + return meta; } /** - * load a blob and check if it is json + * get input + * @param {*} _inputsMeta + * @param {*} _dataRaw */ - async loadBlobInternal() { - try { - const data = await fetch(this.config.dataUrl); - const text = await data.text(); - if (this.isJsonString(text)) { - const json = JSON.parse(text); - await this.loadJSONInternal(json); - } else { - const json = this.csvJSON(text); - await this.loadJSONInternal(json); + // eslint-disable-next-line class-methods-use-this, no-unused-vars + getInputMetaUnits(_dataRaw, _inputsMeta) { + let units = 0; + const inputsMeta = Object.assign({}, _inputsMeta); + + Object.entries(inputsMeta).forEach(arr => { + const { dtype } = arr[1]; + if (dtype === 'number') { + units += 1; + } else if (dtype === 'string') { + const { uniqueValues } = arr[1]; + + const uniqueCount = uniqueValues.length; + units += uniqueCount; + } else if (dtype === 'array') { + // TODO: User must input the shape of the + // image size correctly. + units = []; } - } catch (err) { - console.log('mmm might be passing in a string or something!', err) - } + }); + + return units; } /** - * sets the data types of the data we're using + * getDTypesFromData + * gets the data types of the data we're using * important for handling oneHot */ - setDTypes() { - // this.meta.inputs - const sample = this.data.raw[0]; + getDTypesFromData(_dataRaw) { + const meta = { + ...this.meta, + inputs: {}, + outputs: {}, + }; + + const sample = _dataRaw[0]; const xs = Object.keys(sample.xs); const ys = Object.keys(sample.ys); - xs.forEach((prop) => { - this.meta.inputs[prop] = { - dtype: typeof sample.xs[prop] - } + xs.forEach(prop => { + meta.inputs[prop] = { + dtype: nnUtils.getDataType(sample.xs[prop]), + }; }); - ys.forEach((prop) => { - this.meta.outputs[prop] = { - dtype: typeof sample.ys[prop] - } + ys.forEach(prop => { + meta.outputs[prop] = { + dtype: nnUtils.getDataType(sample.ys[prop]), + }; }); + // TODO: check if all entries have the same dtype. + // otherwise throw an error + + this.meta = meta; + + return meta; } /** - * Get the input and output units - * + * //////////////////////////////////////////////////////// + * Add Data + * //////////////////////////////////////////////////////// */ - getIOUnits() { - let inputUnits = 0; - let outputUnits = 0; - // TODO: turn these into functions! - // calc the number of inputs/output units - Object.entries(this.meta.inputs).forEach(arr => { - const { - dtype - } = arr[1]; - const prop = arr[0]; - if (dtype === 'number') { - inputUnits += 1; - } else if (dtype === 'string') { - const uniqueVals = [...new Set(this.data.raw.map(obj => obj.xs[prop]))] - // Store the unqiue values - this.meta.inputs[prop].uniqueValues = uniqueVals; - const onehotValues = [...new Array(uniqueVals.length).fill(null).map((item, idx) => idx)]; + /** + * Add Data + * @param {object} xInputObj, {key: value}, key must be the name of the property value must be a String, Number, or Array + * @param {*} yInputObj, {key: value}, key must be the name of the property value must be a String, Number, or Array + */ + addData(xInputObj, yInputObj) { + this.data.raw.push({ + xs: xInputObj, + ys: yInputObj, + }); + } - const oneHotEncodedValues = tf.oneHot(tf.tensor1d(onehotValues, 'int32'), uniqueVals.length); - const oneHotEncodedValuesArray = oneHotEncodedValues.arraySync(); + /** + * //////////////////////////////////////////////////////// + * Tensor handling + * //////////////////////////////////////////////////////// + */ - this.meta.inputs[prop].legend = {}; - uniqueVals.forEach((uVal, uIdx) => { - this.meta.inputs[prop].legend[uVal] = oneHotEncodedValuesArray[uIdx] - }); + /** + * convertRawToTensors + * converts array of {xs, ys} to tensors + * @param {*} _dataRaw + * @param {*} meta + */ + // eslint-disable-next-line class-methods-use-this, no-unused-vars + convertRawToTensors(dataRaw) { + const meta = Object.assign({}, this.meta); + const dataLength = dataRaw.length; - // increment the number of inputs/outputs - inputUnits += uniqueVals.length; - } - }) + return tf.tidy(() => { + const inputArr = []; + const outputArr = []; + + dataRaw.forEach(row => { + // get xs + const xs = Object.keys(meta.inputs) + .map(k => { + return row.xs[k]; + }) + .flat(); + + inputArr.push(xs); + + // get ys + const ys = Object.keys(meta.outputs) + .map(k => { + return row.ys[k]; + }) + .flat(); + + outputArr.push(ys); + }); + const inputs = tf.tensor(inputArr.flat(), [dataLength, ...meta.inputUnits]); + const outputs = tf.tensor(outputArr.flat(), [dataLength, meta.outputUnits]); - // calc the number of inputs/output units - Object.entries(this.meta.outputs).forEach(arr => { - const { - dtype - } = arr[1]; - const prop = arr[0]; - if (dtype === 'number') { - outputUnits += 1; - } else if (dtype === 'string') { - const uniqueVals = [...new Set(this.data.raw.map(obj => obj.ys[prop]))] - // Store the unqiue values - this.meta.outputs[prop].uniqueValues = uniqueVals; - const onehotValues = [...new Array(uniqueVals.length).fill(null).map((item, idx) => idx)]; + return { + inputs, + outputs, + }; + }); + } - const oneHotEncodedValues = tf.oneHot(tf.tensor1d(onehotValues, 'int32'), uniqueVals.length); - const oneHotEncodedValuesArray = oneHotEncodedValues.arraySync(); + /** + * //////////////////////////////////////////////////////// + * data normalization / unnormalization + * //////////////////////////////////////////////////////// + */ - this.meta.outputs[prop].legend = {}; - uniqueVals.forEach((uVal, uIdx) => { - this.meta.outputs[prop].legend[uVal] = oneHotEncodedValuesArray[uIdx] - }); + /** + * normalize the dataRaw input + * @param {*} dataRaw + */ + normalizeDataRaw(dataRaw) { + const meta = Object.assign({}, this.meta); - // increment the number of inputs/outputs - outputUnits += uniqueVals.length; - } - }) + const normXs = this.normalizeInputData(dataRaw, meta.inputs, 'xs'); + const normYs = this.normalizeInputData(dataRaw, meta.outputs, 'ys'); + + const normalizedData = nnUtils.zipArrays(normXs, normYs); - this.meta.inputUnits = inputUnits; - this.meta.outputUnits = outputUnits; + return normalizedData; } /** - * Takes in a number or array and then either returns - * the array or returns an array of ['input0','input1'] - * the array or returns an array of ['output0','output1'] - * @param {*} val - * @param {*} inputType + * normalizeRaws + * @param {*} dataRaw + * @param {*} inputOrOutputMeta + * @param {*} xsOrYs */ - // eslint-disable-next-line class-methods-use-this - createNamedIO(val, inputType) { - const arr = (val instanceof Array) ? val : [...new Array(val).fill(null).map((item, idx) => `${inputType}${idx}`)] - return arr; + // eslint-disable-next-line no-unused-vars, class-methods-use-this + normalizeInputData(dataRaw, inputOrOutputMeta, xsOrYs) { + // the data length + const dataLength = dataRaw.length; + // the copy of the inputs.meta[inputOrOutput] + const inputMeta = Object.assign({}, inputOrOutputMeta); + + // normalized output object + const normalized = {}; + Object.keys(inputMeta).forEach(k => { + // get the min and max values + const options = { + min: inputMeta[k].min, + max: inputMeta[k].max, + }; + + const dataAsArray = dataRaw.map(item => item[xsOrYs][k]); + // depending on the input type, normalize accordingly + if (inputMeta[k].dtype === 'string') { + options.legend = inputMeta[k].legend; + normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === 'number') { + normalized[k] = this.normalizeArray(dataAsArray, options); + } else if (inputMeta[k].dtype === 'array') { + normalized[k] = dataAsArray.map(item => this.normalizeArray(item, options)); + } + }); + + // create a normalized version of data.raws + const output = [...new Array(dataLength).fill(null)].map((item, idx) => { + const row = { + [xsOrYs]: {}, + }; + + Object.keys(inputMeta).forEach(k => { + row[xsOrYs][k] = normalized[k][idx]; + }); + + return row; + }); + + return output; } /** - * checks whether or not a string is a json - * @param {*} str + * normalizeArray + * @param {*} _input + * @param {*} _options */ - // eslint-disable-next-line class-methods-use-this - isJsonString(str) { - try { - JSON.parse(str); - } catch (e) { - return false; + // eslint-disable-next-line no-unused-vars, class-methods-use-this + normalizeArray(inputArray, options) { + const { min, max } = options; + + // if the data are onehot encoded, replace the string + // value with the onehot array + // if none exists, return the given value + if (options.legend) { + const normalized = inputArray.map(v => { + return options.legend[v] ? options.legend[v] : v; + }); + return normalized; + } + + // if the dtype is a number + if (inputArray.every(v => typeof v === 'number')) { + const normalized = inputArray.map(v => nnUtils.normalizeValue(v, min, max)); + return normalized; } - return true; - } + // otherwise return the input array + // return inputArray; + throw new Error('error in inputArray of normalizeArray() function'); + } /** - * Creates a csv from a strin - * @param {*} csv + * unNormalizeArray + * @param {*} _input + * @param {*} _options */ - // via: http://techslides.com/convert-csv-to-json-in-javascript - // eslint-disable-next-line class-methods-use-this - csvJSON(csv) { + // eslint-disable-next-line no-unused-vars, class-methods-use-this + unnormalizeArray(inputArray, options) { + const { min, max } = options; + + // if the data is onehot encoded then remap the + // values from those oneHot arrays + if (options.legend) { + const unnormalized = inputArray.map(v => { + let res; + Object.entries(options.legend).forEach(item => { + const key = item[0]; + const val = item[1]; + const matches = v.map((num, idx) => num === val[idx]).every(truthy => truthy === true); + if (matches) res = key; + }); + return res; + }); - const lines = csv.split("\n"); + return unnormalized; + } - const result = []; + // if the dtype is a number + if (inputArray.every(v => typeof v === 'number')) { + const unnormalized = inputArray.map(v => nnUtils.unnormalizeValue(v, min, max)); + return unnormalized; + } - const headers = lines[0].split(","); + // otherwise return the input array + // return inputArray; + throw new Error('error in inputArray of normalizeArray() function'); + } - for (let i = 1; i < lines.length; i += 1) { + /* + * //////////////////////////////////////////////// + * One hot encoding handling + * //////////////////////////////////////////////// + */ - const obj = {}; - const currentline = lines[i].split(","); + /** + * applyOneHotEncodingsToDataRaw + * does not set this.data.raws + * but rather returns them + * @param {*} _dataRaw + * @param {*} _meta + */ + applyOneHotEncodingsToDataRaw(dataRaw) { + const meta = Object.assign({}, this.meta); + + const output = dataRaw.map(row => { + const xs = { + ...row.xs, + }; + const ys = { + ...row.ys, + }; + // get xs + Object.keys(meta.inputs).forEach(k => { + if (meta.inputs[k].legend) { + xs[k] = meta.inputs[k].legend[row.xs[k]]; + } + }); - for (let j = 0; j < headers.length; j += 1) { - obj[headers[j]] = currentline[j]; - } - result.push(obj); - } + Object.keys(meta.outputs).forEach(k => { + if (meta.outputs[k].legend) { + ys[k] = meta.outputs[k].legend[row.ys[k]]; + } + }); - return { - entries: result - } + return { + xs, + ys, + }; + }); + + return output; } /** - * Takes data as an array - * @param {*} xArray - * @param {*} yArray + * getDataOneHot + * creates onehot encodings for the input and outputs + * and adds them to the meta info + * @param {*} dataRaw */ - addData(xInputs, yInputs) { - let inputs = {}; - let outputs = {}; + getDataOneHot(dataRaw) { + const meta = Object.assign({}, this.meta); + const inputMeta = this.getInputMetaOneHot(dataRaw, meta.inputs, 'xs'); + const outputMeta = this.getInputMetaOneHot(dataRaw, meta.outputs, 'ys'); + meta.inputs = inputMeta; + meta.outputs = outputMeta; - if (Array.isArray(xInputs)) { - xInputs.forEach((item, idx) => { - // TODO: get the label from the inputs? - // const label = `input${idx}`; - const label = this.config.dataOptions.inputs[idx]; - inputs[label] = item; - }); - } else if (typeof xInputs === 'object') { - inputs = xInputs; - } else { - console.log('input provided is not supported or does not match your output label specifications'); - } - - if (Array.isArray(yInputs)) { - yInputs.forEach((item, idx) => { - // TODO: get the label from the outputs? - // const label = `output${idx}`; - const label = this.config.dataOptions.outputs[idx]; - outputs[label] = item; - }); - } else if (typeof yInputs === 'object') { - outputs = yInputs; - } else { - console.log('input provided is not supported or does not match your output label specifications'); - } + this.meta = { + ...this.meta, + ...meta, + }; - this.data.raw.push({ - xs: inputs, - ys: outputs - }); + return meta; } /** - * normalize the data.raw - */ - normalize() { - // always make sure to check set the data types - this.setDTypes(); - // always make sure that the IO units are set - this.getIOUnits(); - - // do the things...! - const { - inputTensor, - outputTensor - } = this.convertRawToTensor(); - - // run normalize on the new tensors - const { - normalizedInputs, - normalizedOutputs, - inputMax, - inputMin, - outputMax, - outputMin - } = this.normalizeInternal(inputTensor, outputTensor); - - // set the tensor data to the normalized inputs - this.data.tensor = { - inputs: normalizedInputs, - outputs: normalizedOutputs, - inputMax, - inputMin, - outputMax, - outputMin, + * getOneHotMeta + * @param {*} _inputsMeta + * @param {*} _dataRaw + * @param {*} xsOrYs + */ + getInputMetaOneHot(_dataRaw, _inputsMeta, xsOrYs) { + const inputsMeta = Object.assign({}, _inputsMeta); + + Object.entries(inputsMeta).forEach(arr => { + // the key + const key = arr[0]; + // the value + const { dtype } = arr[1]; + + if (dtype === 'string') { + const uniqueVals = [...new Set(_dataRaw.map(obj => obj[xsOrYs][key]))]; + const oneHotMeta = this.createOneHotEncodings(uniqueVals); + inputsMeta[key] = { + ...inputsMeta[key], + ...oneHotMeta, + }; } + }); - // set the input/output Min and max values as numbers - this.data.inputMin = inputMin.arraySync(); - this.data.inputMax = inputMax.arraySync(); - this.data.outputMax = outputMax.arraySync(); - this.data.outputMin = outputMin.arraySync(); - - // set isNormalized to true if this function is called - this.meta.isNormalized = true; + return inputsMeta; } /** - * + * Returns a legend mapping the + * data values to oneHot encoded values */ - normalizeInternal(inputTensor, outputTensor) { + // eslint-disable-next-line class-methods-use-this, no-unused-vars + createOneHotEncodings(_uniqueValuesArray) { return tf.tidy(() => { - // inputTensor.print() - // outputTensor.print() - - // 4. Get the min and max values for normalization - // TODO: allow people to submit their own normalization values! - const { - inputMax, - inputMin, - outputMax, - outputMin - } = this.setIOStats(inputTensor, outputTensor); - - // 5. create a normalized tensor - const normalizedInputs = inputTensor.sub(inputMin).div(inputMax.sub(inputMin)); - const normalizedOutputs = outputTensor.sub(outputMin).div(outputMax.sub(outputMin)); + const output = { + uniqueValues: _uniqueValuesArray, + legend: {}, + }; + + const uniqueVals = _uniqueValuesArray; // [...new Set(this.data.raw.map(obj => obj.xs[prop]))] + // get back values from 0 to the length of the uniqueVals array + const onehotValues = uniqueVals.map((item, idx) => idx); + // oneHot encode the values in the 1d tensor + const oneHotEncodedValues = tf.oneHot(tf.tensor1d(onehotValues, 'int32'), uniqueVals.length); + // convert them from tensors back out to an array + const oneHotEncodedValuesArray = oneHotEncodedValues.arraySync(); + + // populate the legend with the key/values + uniqueVals.forEach((uVal, uIdx) => { + output.legend[uVal] = oneHotEncodedValuesArray[uIdx]; + }); - return { - normalizedInputs, - normalizedOutputs, - inputMin, - inputMax, - outputMax, - outputMin + return output; + }); + } + + /** + * //////////////////////////////////////////////// + * saving / loading data + * //////////////////////////////////////////////// + */ + + /** + * Loads data from a URL using the appropriate function + * @param {*} dataUrl + * @param {*} inputs + * @param {*} outputs + */ + async loadDataFromUrl(dataUrl, inputs, outputs) { + try { + let result; + + if (dataUrl.endsWith('.csv')) { + result = await this.loadCSV(dataUrl, inputs, outputs); + } else if (dataUrl.endsWith('.json')) { + result = await this.loadJSON(dataUrl, inputs, outputs); + } else if (dataUrl.includes('blob')) { + result = await this.loadBlob(dataUrl, inputs, outputs); + } else { + throw new Error('Not a valid data format. Must be csv or json'); } - }) + + return result; + } catch (error) { + console.error(error); + throw new Error(error); + } } /** - * Assemble the data for training + * loadJSON + * @param {*} _dataUrlOrJson + * @param {*} _inputLabelsArray + * @param {*} _outputLabelsArray */ - warmUp() { - return tf.tidy(() => { - // always make sure to check set the data types - this.setDTypes(); - // always make sure that the IO units are set - this.getIOUnits(); - - // do the things...! - const { - inputTensor, - outputTensor - } = this.convertRawToTensor(); - - const { - inputMax, - inputMin, - outputMax, - outputMin - } = this.setIOStats(inputTensor, outputTensor); - - this.data.tensor = { - inputs: inputTensor, - outputs: outputTensor, - inputMax, - inputMin, - outputMax, - outputMin, + async loadJSON(dataUrlOrJson, inputLabels, outputLabels) { + try { + let json; + // handle loading parsedJson + if (dataUrlOrJson instanceof Object) { + json = Object.assign({}, dataUrlOrJson); + } else { + const data = await fetch(dataUrlOrJson); + json = await data.json(); } - // set the input/output Min and max values as numbers - this.data.inputMin = inputMin.arraySync(); - this.data.inputMax = inputMax.arraySync(); - this.data.outputMax = outputMax.arraySync(); - this.data.outputMin = outputMin.arraySync(); + // format the data.raw array + const result = this.formatRawData(json, inputLabels, outputLabels); + return result; + } catch (err) { + console.error('error loading json'); + throw new Error(err); + } + } - }) + /** + * loadCSV + * @param {*} _dataUrl + * @param {*} _inputLabelsArray + * @param {*} _outputLabelsArray + */ + async loadCSV(dataUrl, inputLabels, outputLabels) { + try { + const myCsv = tf.data.csv(dataUrl); + const loadedData = await myCsv.toArray(); + const json = { + entries: loadedData, + }; + // format the data.raw array + const result = this.formatRawData(json, inputLabels, outputLabels); + return result; + } catch (err) { + console.error('error loading csv', err); + throw new Error(err); + } } /** - * get the min and max values of the input and output data - * @param {*} inputTensor - * @param {*} outputTensor + * loadBlob + * @param {*} _dataUrlOrJson + * @param {*} _inputLabelsArray + * @param {*} _outputLabelsArray */ - setIOStats(inputTensor, outputTensor) { - return tf.tidy(() => { - let inputMax; - let inputMin; - let outputMax; - let outputMin; - // TODO: this is terrible and can be handled way better! - // REFACTOR THIS!!! - if (this.config.architecture.task === 'regression') { - if (this.config.dataOptions.normalizationOptions instanceof Object) { - // if there is an object with normalizationOptions - inputMax = this.data.inputMax !== null ? tf.tensor1d(this.data.inputMax) : inputTensor.max(0); - inputMin = this.data.inputMin !== null ? tf.tensor1d(this.data.inputMin) : inputTensor.min(0); - outputMax = this.data.outputMax !== null ? tf.tensor1d(this.data.outputMax) : outputTensor.max(0); - outputMin = this.data.outputMin !== null ? tf.tensor1d(this.data.outputMin) : outputTensor.min(0); + async loadBlob(dataUrlOrJson, inputLabels, outputLabels) { + try { + const data = await fetch(dataUrlOrJson); + const text = await data.text(); + + let result; + if (nnUtils.isJsonOrString(text)) { + const json = JSON.parse(text); + result = await this.loadJSON(json, inputLabels, outputLabels); + } else { + const json = this.csvToJSON(text); + result = await this.loadJSON(json, inputLabels, outputLabels); + } + + return result; + } catch (err) { + console.log('mmm might be passing in a string or something!', err); + throw new Error(err); + } + } + + /** + * loadData from fileinput or path + * @param {*} filesOrPath + * @param {*} callback + */ + async loadData(filesOrPath = null, callback) { + try { + let loadedData; + + if (typeof filesOrPath !== 'string') { + const file = filesOrPath[0]; + const fr = new FileReader(); + fr.readAsText(file); + if (file.name.includes('.json')) { + const temp = await file.text(); + loadedData = JSON.parse(temp); } else { - // if the task is a regression, return all the - // output stats as an array - inputMax = inputTensor.max(0); - inputMin = inputTensor.min(0); - outputMax = outputTensor.max(0); - outputMin = outputTensor.min(0); + console.log('data must be a json object containing an array called "data" or "entries'); } - } else if (this.config.architecture.task === 'classification') { - if (this.config.dataOptions.normalizationOptions instanceof Object) { - // if there is an object with normalizationOptions - inputMax = this.data.inputMax !== null ? tf.tensor1d(this.data.inputMax) : inputTensor.max(0); - inputMin = this.data.inputMin !== null ? tf.tensor1d(this.data.inputMin) : inputTensor.min(0); - outputMax = this.data.outputMax !== null ? tf.tensor1d(this.data.outputMax) : outputTensor.max(); - outputMin = this.data.outputMin !== null ? tf.tensor1d(this.data.outputMin) : outputTensor.min(); + } else { + loadedData = await fetch(filesOrPath); + const text = await loadedData.text(); + if (nnUtils.isJsonOrString(text)) { + loadedData = JSON.parse(text); } else { - // if the task is a classification, return the single value - inputMax = inputTensor.max(0); - inputMin = inputTensor.min(0); - outputMax = outputTensor.max(); - outputMin = outputTensor.min(); + console.log( + 'Whoops! something went wrong. Either this kind of data is not supported yet or there is an issue with .loadData', + ); } } - // return the values calculated here - return { - inputMax, - inputMin, - outputMax, - outputMin + + this.data.raw = this.findEntries(loadedData); + + // check if a data or entries property exists + if (!this.data.raw.length > 0) { + console.log('data must be a json object containing an array called "data" '); } - }) + + if (callback) { + callback(); + } + } catch (error) { + throw new Error(error); + } } /** - * onehot encode values + * saveData + * @param {*} name */ - convertRawToTensor() { - return tf.tidy(() => { - // console.log(this.meta) - - // Given the inputs and output types, - // now create the input and output tensors - // 1. start by creating a matrix - const inputs = [] - const outputs = []; - - // 2. encode the values - // iterate through each entry and send the correct - // oneHot encoded values or the numeric value - this.data.raw.forEach((item) => { - let inputRow = []; - let outputRow = []; - const { - xs, - ys - } = item; - - // Create the inputs matrix - Object.entries(xs).forEach((valArray) => { - const prop = valArray[0]; - const val = valArray[1]; - const { - dtype - } = this.meta.inputs[prop]; - - if (dtype === 'number') { - inputRow.push(val); - } else if (dtype === 'string') { - const oneHotArray = this.meta.inputs[prop].legend[val]; - inputRow = [...inputRow, ...oneHotArray]; - } + async saveData(name) { + const today = new Date(); + const date = `${String(today.getFullYear())}-${String(today.getMonth() + 1)}-${String( + today.getDate(), + )}`; + const time = `${String(today.getHours())}-${String(today.getMinutes())}-${String( + today.getSeconds(), + )}`; + const datetime = `${date}_${time}`; - }); + let dataName = datetime; + if (name) dataName = name; + + const output = { + data: this.data.raw, + }; + + await saveBlob(JSON.stringify(output), `${dataName}.json`, 'text/plain'); + } - // Create the outputs matrix - Object.entries(ys).forEach((valArray) => { - const prop = valArray[0]; - const val = valArray[1]; - const { - dtype - } = this.meta.outputs[prop]; - - if (dtype === 'number') { - outputRow.push(val); - } else if (dtype === 'string') { - const oneHotArray = this.meta.outputs[prop].legend[val]; - outputRow = [...outputRow, ...oneHotArray]; + /** + * Saves metadata of the data + * @param {*} nameOrCb + * @param {*} cb + */ + async saveMeta(nameOrCb, cb) { + let modelName; + let callback; + + if (typeof nameOrCb === 'function') { + modelName = 'model'; + callback = nameOrCb; + } else if (typeof nameOrCb === 'string') { + modelName = nameOrCb; + + if (typeof cb === 'function') { + callback = cb; + } + } else { + modelName = 'model'; + } + + await saveBlob(JSON.stringify(this.meta), `${modelName}_meta.json`, 'text/plain'); + if (callback) { + callback(); + } + } + + /** + * load a model and metadata + * @param {*} filesOrPath + * @param {*} callback + */ + async loadMeta(filesOrPath = null, callback) { + if (filesOrPath instanceof FileList) { + const files = await Promise.all( + Array.from(filesOrPath).map(async file => { + if (file.name.includes('.json') && !file.name.includes('_meta')) { + return { + name: 'model', + file, + }; + } else if (file.name.includes('.json') && file.name.includes('_meta.json')) { + const modelMetadata = await file.text(); + return { + name: 'metadata', + file: modelMetadata, + }; + } else if (file.name.includes('.bin')) { + return { + name: 'weights', + file, + }; } + return { + name: null, + file: null, + }; + }), + ); - }); + const modelMetadata = JSON.parse(files.find(item => item.name === 'metadata').file); - inputs.push(inputRow); - outputs.push(outputRow); + this.meta = modelMetadata; + } else if (filesOrPath instanceof Object) { + // filesOrPath = {model: URL, metadata: URL, weights: URL} - }); + let modelMetadata = await fetch(filesOrPath.metadata); + modelMetadata = await modelMetadata.text(); + modelMetadata = JSON.parse(modelMetadata); - // console.log(inputs, outputs) - // 3. convert to tensors - const inputTensor = tf.tensor(inputs, [this.data.raw.length, this.meta.inputUnits]); - const outputTensor = tf.tensor(outputs, [this.data.raw.length, this.meta.outputUnits]); + this.meta = modelMetadata; + } else { + const metaPath = `${filesOrPath.substring(0, filesOrPath.lastIndexOf('/'))}/model_meta.json`; + let modelMetadata = await fetch(metaPath); + modelMetadata = await modelMetadata.json(); - return { - inputTensor, - outputTensor - } + this.meta = modelMetadata; + } - }) + this.isMetadataReady = true; + this.isWarmedUp = true; + + if (callback) { + callback(); + } + return this.meta; } + /* + * //////////////////////////////////////////////// + * data loading helpers + * //////////////////////////////////////////////// + */ - async saveData(name) { - const today = new Date(); - const date = `${String(today.getFullYear())}-${String(today.getMonth()+1)}-${String(today.getDate())}`; - const time = `${String(today.getHours())}-${String(today.getMinutes())}-${String(today.getSeconds())}`; - const datetime = `${date}_${time}`; + /** + * // TODO: convert ys into strings, if the task is classification + // if (this.config.architecture.task === "classification" && typeof output.ys[prop] !== "string") { + // output.ys[prop] += ""; + // } + * formatRawData + * takes a json and set the this.data.raw + * @param {*} json + * @param {Array} inputLabels + * @param {Array} outputLabels + */ + formatRawData(json, inputLabels, outputLabels) { + // Recurse through the json object to find + // an array containing `entries` or `data` + const dataArray = this.findEntries(json); - let dataName = datetime; - if (name) dataName = name; + if (!dataArray.length > 0) { + console.log(`your data must be contained in an array in \n + a property called 'entries' or 'data' of your json object`); + } - const output = { - data: this.data.raw + // create an array of json objects [{xs,ys}] + const result = dataArray.map((item, idx) => { + const output = { + xs: {}, + ys: {}, + }; + + inputLabels.forEach(k => { + if (item[k] !== undefined) { + output.xs[k] = item[k]; + } else { + console.error(`the input label ${k} does not exist at row ${idx}`); + } + }); + + outputLabels.forEach(k => { + if (item[k] !== undefined) { + output.ys[k] = item[k]; + } else { + console.error(`the output label ${k} does not exist at row ${idx}`); + } + }); + + return output; + }); + + // set this.data.raw + this.data.raw = result; + + return result; + } + + /** + * csvToJSON + * Creates a csv from a string + * @param {*} csv + */ + // via: http://techslides.com/convert-csv-to-json-in-javascript + // eslint-disable-next-line class-methods-use-this + csvToJSON(csv) { + // split the string by linebreak + const lines = csv.split('\n'); + const result = []; + // get the header row as an array + const headers = lines[0].split(','); + + // iterate through every row + for (let i = 1; i < lines.length; i += 1) { + // create a json object for each row + const row = {}; + // split the current line into an array + const currentline = lines[i].split(','); + + // for each header, create a key/value pair + headers.forEach((k, idx) => { + row[k] = currentline[idx]; + }); + // add this to the result array + result.push(row); } - await saveBlob(JSON.stringify(output), `${dataName}.json`, 'text/plain'); + return { + entries: result, + }; } + /** + * findEntries + * recursively attempt to find the entries + * or data array for the given json object + * @param {*} _data + */ + findEntries(_data) { + const parentCopy = Object.assign({}, _data); + + if (parentCopy.entries && parentCopy.entries instanceof Array) { + return parentCopy.entries; + } else if (parentCopy.data && parentCopy.data instanceof Array) { + return parentCopy.data; + } -} // end of class + const keys = Object.keys(parentCopy); + // eslint-disable-next-line consistent-return + keys.forEach(k => { + if (typeof parentCopy[k] === 'object') { + return this.findEntries(parentCopy[k]); + } + }); + return parentCopy; + } +} -export default NeuralNetworkData; \ No newline at end of file +export default NeuralNetworkData; diff --git a/src/NeuralNetwork/NeuralNetworkDefaults.js b/src/NeuralNetwork/NeuralNetworkDefaults.js deleted file mode 100644 index b6c4c610b..000000000 --- a/src/NeuralNetwork/NeuralNetworkDefaults.js +++ /dev/null @@ -1,19 +0,0 @@ -const DEFAULTS = { - task: 'regression', - activationHidden: 'sigmoid', - activationOutput: 'sigmoid', - debug: false, - learningRate: 0.25, - inputs: 2, - outputs: 1, - noVal: null, - hiddenUnits: 16, - modelMetrics: ['accuracy'], - modelLoss: 'meanSquaredError', - modelOptimizer: null, - batchSize: 64, - epochs: 32, - returnTensors: false, -} - -export default DEFAULTS; \ No newline at end of file diff --git a/src/NeuralNetwork/NeuralNetworkUtils.js b/src/NeuralNetwork/NeuralNetworkUtils.js new file mode 100644 index 000000000..870a4a12e --- /dev/null +++ b/src/NeuralNetwork/NeuralNetworkUtils.js @@ -0,0 +1,151 @@ +class NeuralNetworkUtils { + constructor(options) { + this.options = options || {}; + } + + /** + * normalizeValue + * @param {*} value + * @param {*} min + * @param {*} max + */ + // eslint-disable-next-line class-methods-use-this + normalizeValue(value, min, max) { + return ((value - min) / (max - min)) + } + + /** + * unNormalizeValue + * @param {*} value + * @param {*} min + * @param {*} max + */ + // eslint-disable-next-line class-methods-use-this + unnormalizeValue(value, min, max) { + return ((value * (max - min)) + min) + } + + /** + * getMin + * @param {*} _array + */ + // eslint-disable-next-line no-unused-vars, class-methods-use-this + getMin(_array) { + // return Math.min(..._array) + return _array.reduce((a, b) => { + return Math.min(a, b); + }); + } + + /** + * getMax + * @param {*} _array + */ + // eslint-disable-next-line no-unused-vars, class-methods-use-this + getMax(_array) { + return _array.reduce((a, b) => { + return Math.max(a, b); + }); + // return Math.max(..._array) + } + + /** + * checks whether or not a string is a json + * @param {*} str + */ + // eslint-disable-next-line class-methods-use-this + isJsonOrString(str) { + try { + JSON.parse(str); + } catch (e) { + return false; + } + return true; + } + + /** + * zipArrays + * @param {*} arr1 + * @param {*} arr2 + */ + // eslint-disable-next-line no-unused-vars, class-methods-use-this + zipArrays(arr1, arr2) { + if (arr1.length !== arr2.length) { + console.error('arrays do not have the same length') + return []; + } + + const output = [...new Array(arr1.length).fill(null)].map((item, idx) => { + return { + ...arr1[idx], + ...arr2[idx] + } + }) + + return output; + } + + /** + * createLabelsFromArrayValues + * @param {*} incoming + * @param {*} prefix + */ + // eslint-disable-next-line class-methods-use-this + createLabelsFromArrayValues(incoming, prefix) { + let labels; + if (Array.isArray(incoming)) { + labels = incoming.map((v, idx) => `${prefix}_${idx}`) + } + return labels; + } + + /** + * takes an array and turns it into a json object + * where the labels are the keys and the array values + * are the object values + * @param {*} incoming + * @param {*} labels + */ + // eslint-disable-next-line class-methods-use-this + formatDataAsObject(incoming, labels) { + let result = {}; + + if (Array.isArray(incoming)) { + incoming.forEach((item, idx) => { + const label = labels[idx]; + result[label] = item; + }); + return result; + } else if (typeof incoming === 'object') { + result = incoming; + return result; + } + + throw new Error('input provided is not supported or does not match your output label specifications') + } + + /** + * returns a datatype of the value as string + * @param {*} val + */ + // eslint-disable-next-line class-methods-use-this + getDataType(val) { + let dtype = typeof val; + + if (dtype === 'object') { + if (Array.isArray(val)) { + dtype = 'array' + } + } + + return dtype; + } + +} + +const neuralNetworkUtils = () => { + const instance = new NeuralNetworkUtils(); + return instance; +} + +export default neuralNetworkUtils(); \ No newline at end of file diff --git a/src/NeuralNetwork/NeuralNetworkVis.js b/src/NeuralNetwork/NeuralNetworkVis.js index e24d80cad..5599a6988 100644 --- a/src/NeuralNetwork/NeuralNetworkVis.js +++ b/src/NeuralNetwork/NeuralNetworkVis.js @@ -10,9 +10,12 @@ class NeuralNetworkVis { height: 300, }; - // store tfvis here for now so people can access it - // through ml5? - this.tfvis = tfvis; + } + + // eslint-disable-next-line class-methods-use-this + modelSummary(_options, _model){ + const options = {..._options}; + tfvis.show.modelSummary(options, _model); } /** diff --git a/src/NeuralNetwork/index.js b/src/NeuralNetwork/index.js index 874ccd8e1..4425b3267 100644 --- a/src/NeuralNetwork/index.js +++ b/src/NeuralNetwork/index.js @@ -1,304 +1,437 @@ -// Copyright (c) 2019 ml5 -// -// This software is released under the MIT License. -// https://opensource.org/licenses/MIT - -/* -Generic NeuralNetwork class -*/ - import * as tf from '@tensorflow/tfjs'; -import callCallback from '../utils/callcallback'; -import { - saveBlob -} from '../utils/io'; -// import { input } from '@tensorflow/tfjs'; -import DEFAULTS from './NeuralNetworkDefaults'; +import NeuralNetwork from './NeuralNetwork'; import NeuralNetworkData from './NeuralNetworkData'; import NeuralNetworkVis from './NeuralNetworkVis'; +import callCallback from '../utils/callcallback'; -class NeuralNetwork { - /** - * Create a Neural Network. - * @param {object} options - An object with options. - */ - constructor(options, callback) { - // model config - this.config = { - // debugging - debug: options.debug || DEFAULTS.debug, - returnTensors: options.returnTensors || DEFAULTS.returnTensors, - // architecture - architecture: { - task: options.task || DEFAULTS.task, - // array of layers, the last is always the output layer - layers: options.layers || [], - // array of activations corresponding to the layer number - activations: [], - // hiddenUnits - hiddenUnits: options.hiddenUnits || DEFAULTS.hiddenUnits, - // the units of the layers will come from the config.dataOptions - }, - training: { - // defined either on instantiation or in .train(options) - batchSize: options.batchSize || DEFAULTS.batchSize, - epochs: options.epochs || DEFAULTS.epochs, - // will depend on the config.architecture.task - learningRate: options.learningRate || DEFAULTS.learningRate, - modelMetrics: options.modelMetrics || DEFAULTS.modelMetrics, - modelLoss: options.modelLoss || DEFAULTS.modelLoss, - modelOptimizer: options.modelOptimizer || DEFAULTS.modelOptimizer, - }, - // data - dataOptions: { - dataUrl: options.dataUrl || null, - inputs: options.inputs || DEFAULTS.inputs, - outputs: options.outputs || DEFAULTS.outputs, - // TODO: adding option for normalization - normalizationOptions: options.normalizationOptions || null - }, - - } - - - // TODO: maybe we create a set of configs for - // regression vs. classification - // set the default activations: - if (this.config.architecture.task === 'regression') { - // current defaults are for regression - const activationHidden = options.activationHidden || DEFAULTS.activationHidden; - const activationOutput = options.activationOutput || DEFAULTS.activationOutput; - - this.config.training.modelOptimizer = options.modelOptimizer || tf.train.adam(this.config.training.learningRate); - this.config.architecture.activations = [activationHidden, activationOutput]; - } else if (this.config.architecture.task === 'classification') { - // set classification specs different from regression in DEFAULTS - const activationHidden = options.activationHidden || DEFAULTS.activationHidden; - const activationOutput = options.activationOutput || 'softmax'; - - this.config.architecture.activations = [activationHidden, activationOutput]; - this.config.training.modelLoss = options.modelLoss || 'categoricalCrossentropy'; - this.config.training.modelOptimizer = options.modelOptimizer || tf.train.sgd(this.config.training.learningRate); - } else { - console.log(`task not defined. please set task: classification OR regression`); - } +import nnUtils from './NeuralNetworkUtils'; +import { imgToPixelArray, isInstanceOfSupportedElement } from '../utils/imageUtilities'; + +const DEFAULTS = { + inputs: [], + outputs: [], + dataUrl: null, + modelUrl: null, + layers: [], + task: null, + debug: false, + learningRate: 0.2, + hiddenUnits: 16, +}; +class DiyNeuralNetwork { + constructor(options, cb) { + this.callback = cb; + this.options = + { + ...DEFAULTS, + ...options, + } || DEFAULTS; + + this.neuralNetwork = new NeuralNetwork(); + this.neuralNetworkData = new NeuralNetworkData(); + this.neuralNetworkVis = new NeuralNetworkVis(); + + this.data = { + training: [], + }; - // vis class - this.vis = new NeuralNetworkVis(); - // data class - this.data = new NeuralNetworkData(this.config); - // check if the model is ready this.ready = false; - // the model - this.model = null; - - // initialize - this.init(callback); + // Methods + this.init = this.init.bind(this); + // adding data + this.addData = this.addData.bind(this); + this.loadDataFromUrl = this.loadDataFromUrl.bind(this); + this.loadDataInternal = this.loadDataInternal.bind(this); + // metadata prep + this.createMetaData = this.createMetaData.bind(this); + // data prep and handling + this.prepareForTraining = this.prepareForTraining.bind(this); + this.normalizeData = this.normalizeData.bind(this); + this.normalizeInput = this.normalizeInput.bind(this); + this.searchAndFormat = this.searchAndFormat.bind(this); + this.formatInputItem = this.formatInputItem.bind(this); + this.convertTrainingDataToTensors = this.convertTrainingDataToTensors.bind(this); + this.formatInputsForPrediction = this.formatInputsForPrediction.bind(this); + this.formatInputsForPredictionAll = this.formatInputsForPredictionAll.bind(this); + this.isOneHotEncodedOrNormalized = this.isOneHotEncodedOrNormalized.bind(this); + // model prep + this.train = this.train.bind(this); + this.trainInternal = this.trainInternal.bind(this); + this.addLayer = this.addLayer.bind(this); + this.createNetworkLayers = this.createNetworkLayers.bind(this); + this.addDefaultLayers = this.addDefaultLayers.bind(this); + this.compile = this.compile.bind(this); + // prediction / classification + this.predict = this.predict.bind(this); + this.predictMultiple = this.predictMultiple.bind(this); + this.classify = this.classify.bind(this); + this.classifyMultiple = this.classifyMultiple.bind(this); + this.predictInternal = this.predictInternal.bind(this); + this.classifyInternal = this.classifyInternal.bind(this); + // save / load data + this.saveData = this.saveData.bind(this); + this.loadData = this.loadData.bind(this); + // save / load model + this.save = this.save.bind(this); + this.load = this.load.bind(this); + + // Initialize + this.init(this.callback); } /** - * ---------------------------------------- - * --- model creation / initialization ---- - * ---------------------------------------- + * //////////////////////////////////////////////////////////// + * Initialization + * //////////////////////////////////////////////////////////// */ /** - * Initialize the model creation + * init + * @param {*} callback */ init(callback) { - // Create the model based on data or the inputs/outputs - if (this.config.dataOptions.dataUrl !== null) { - this.ready = this.createModelFromData(callback); + if (this.options.dataUrl !== null) { + this.ready = this.loadDataFromUrl(this.options, callback); + } else if (this.options.modelUrl !== null) { + // will take a URL to model.json, an object, or files array + this.ready = this.load(this.options.modelUrl, callback); } else { - // --- set the input/output units --- - const inputIOUnits = this.initializeIOUnits(this.config.dataOptions.inputs, 'inputs'); - this.data.meta.inputUnits = inputIOUnits.units; - this.data.config.dataOptions.inputs = inputIOUnits.labels; - - const outputIOUnits = this.initializeIOUnits(this.config.dataOptions.outputs, 'outputs'); - this.data.meta.outputUnits = outputIOUnits.units; - this.data.config.dataOptions.outputs = outputIOUnits.labels; - this.ready = true; } } /** - * if the inputs is a number - * then set the inputUnits as the number - * and then create an array of input labels - * if not, then use what is given - * @param {*} input - * @param {*} ioType + * //////////////////////////////////////////////////////////// + * Adding Data + * //////////////////////////////////////////////////////////// */ - initializeIOUnits(input, ioType) { - let units; - let labels; - let ioLabel; - if (ioType === 'outputs') { - ioLabel = 'output' + /** + * addData + * @param {Array | Object} xInputs + * @param {Array | Object} yInputs + * @param {*} options + */ + addData(xInputs, yInputs, options = null) { + const { inputs, outputs } = this.options; + + // get the input and output labels + // or infer them from the data + let inputLabels; + let outputLabels; + + if (options !== null) { + // eslint-disable-next-line prefer-destructuring + inputLabels = options.inputLabels; + // eslint-disable-next-line prefer-destructuring + outputLabels = options.outputLabels; + } else if (inputs.length > 0 && outputs.length > 0) { + // if the inputs and outputs labels have been defined + // in the constructor + if (inputs.every(item => typeof item === 'string')) { + inputLabels = inputs; + } + if (outputs.every(item => typeof item === 'string')) { + outputLabels = outputs; + } + } else if (typeof xInputs === 'object' && typeof yInputs === 'object') { + inputLabels = Object.keys(xInputs); + outputLabels = Object.keys(yInputs); } else { - ioLabel = 'input' + inputLabels = nnUtils.createLabelsFromArrayValues(xInputs, 'input'); + outputLabels = nnUtils.createLabelsFromArrayValues(yInputs, 'output'); } - if (typeof input === 'number') { - units = input; - labels = this.data.createNamedIO(input, ioLabel); - } else if (Array.isArray(input)) { - units = input.length; - labels = input; - } else { - console.log(`${ioType} in this format are not supported`) + // Make sure that the inputLabels and outputLabels are arrays + if (!(inputLabels instanceof Array)) { + throw new Error('inputLabels must be an array'); + } + if (!(outputLabels instanceof Array)) { + throw new Error('outputLabels must be an array'); } - return { - units, - labels - }; + const formattedInputs = this.searchAndFormat(xInputs); + const xs = nnUtils.formatDataAsObject(formattedInputs, inputLabels); + + const ys = nnUtils.formatDataAsObject(yInputs, outputLabels); + this.neuralNetworkData.addData(xs, ys); } /** - * create Model + * loadData + * @param {*} options + * @param {*} callback */ - createModel() { + loadDataFromUrl(options, callback) { + return callCallback(this.loadDataInternal(options), callback); + } - switch (this.config.architecture.task) { - case 'regression': - // if the layers are not defined default to a - // neuralnet with 2 layers - this.defineModelLayers(); - return this.createModelInternal(); - case 'classification': - // if the layers are not defined default to a - // neuralnet with 2 layers - this.defineModelLayers(); - return this.createModelInternal(); - default: - console.log('no model exists for this type of task yet!'); - return tf.sequential(); + /** + * loadDataInternal + * @param {*} options + */ + async loadDataInternal(options) { + const { dataUrl, inputs, outputs } = options; + + const data = await this.neuralNetworkData.loadDataFromUrl(dataUrl, inputs, outputs); + + // once the data are loaded, create the metadata + // and prep the data for training + // if the inputs are defined as an array of [img_width, img_height, channels] + this.createMetadata(data); + + this.prepareForTraining(data); + } + + /** + * //////////////////////////////////////////////////////////// + * Metadata prep + * //////////////////////////////////////////////////////////// + */ + + createMetaData(dataRaw) { + const { inputs } = this.options; + + let inputShape; + if (Array.isArray(inputs) && inputs.length > 0) { + inputShape = + inputs.every(item => typeof item === 'number') && inputs.length > 0 ? inputs : null; } + + this.neuralNetworkData.createMetadata(dataRaw, inputShape); } /** - * Define the model layers + * //////////////////////////////////////////////////////////// + * Data prep and handling + * //////////////////////////////////////////////////////////// */ - defineModelLayers() { - if (!this.config.architecture.layers.length > 0) { - this.config.architecture.layers = []; - const { - activations, - hiddenUnits - } = this.config.architecture; + /** + * Prepare data for training by applying oneHot to raw + * @param {*} dataRaw + */ + prepareForTraining(_dataRaw = null) { + const dataRaw = _dataRaw === null ? this.neuralNetworkData.data.raw : _dataRaw; + const unnormalizedTrainingData = this.neuralNetworkData.applyOneHotEncodingsToDataRaw(dataRaw); + this.data.training = unnormalizedTrainingData; + this.neuralNetworkData.isWarmedUp = true; - const hidden = tf.layers.dense({ - units: hiddenUnits, - inputShape: [this.data.meta.inputUnits], - activation: activations[0], - }); + return unnormalizedTrainingData; + } - const output = tf.layers.dense({ - units: this.data.meta.outputUnits, - activation: activations[1], - }); + /** + * normalizeData + * @param {*} _dataRaw + * @param {*} _meta + */ + normalizeData(_dataRaw = null) { + const dataRaw = _dataRaw === null ? this.neuralNetworkData.data.raw : _dataRaw; - this.config.architecture.layers = [hidden, output]; + if (!this.neuralNetworkData.isMetadataReady) { + // if the inputs are defined as an array of [img_width, img_height, channels] + this.createMetaData(dataRaw); + } + if (!this.neuralNetworkData.isWarmedUp) { + this.prepareForTraining(dataRaw); } - } + const trainingData = this.neuralNetworkData.normalizeDataRaw(dataRaw); - createModelInternal() { - const model = tf.sequential(); + // set this equal to the training data + this.data.training = trainingData; - // add the layers to the model as defined in config.architecture.layers - this.config.architecture.layers.forEach(layer => { - model.add(layer); - }); + // set isNormalized to true + this.neuralNetworkData.meta.isNormalized = true; - // compile the model - const { - modelOptimizer, - modelLoss, - modelMetrics - } = this.config.training; - - model.compile({ - optimizer: modelOptimizer, - loss: modelLoss, - metrics: modelMetrics, - }); + return trainingData; + } - return model; + /** + * normalize the input value + * @param {*} value + * @param {*} _key + * @param {*} _meta + */ + // eslint-disable-next-line class-methods-use-this + normalizeInput(value, _key, _meta) { + const key = _key; + const { min, max } = _meta[key]; + return nnUtils.normalizeValue(value, min, max); } /** - * create model from data - * @param {*} callback + * search though the xInputs and format for adding to data.raws + * @param {*} input */ - createModelFromData(callback) { - return callCallback(this.createModelFromDataInternal(), callback) + searchAndFormat(input) { + let formattedInputs; + if (Array.isArray(input)) { + formattedInputs = input.map(item => this.formatInputItem(item)); + } else if (typeof input === 'object') { + const newXInputs = Object.assign({}, input); + Object.keys(input).forEach(k => { + const val = input[k]; + newXInputs[k] = this.formatInputItem(val); + }); + formattedInputs = newXInputs; + } + return formattedInputs; } /** - * Creates model architecture from the loaded data + * Returns either the original input or a pixelArray[] + * @param {*} input */ - async createModelFromDataInternal() { - // load the data - await this.data.loadData(); - // check the input columns for data type to - // calculate the total number of inputs - // and outputs - this.data.getIOUnits(); + // eslint-disable-next-line class-methods-use-this + formatInputItem(input) { + let imgToPredict; + let formattedInputs; + if (isInstanceOfSupportedElement(input)) { + imgToPredict = input; + } else if (typeof input === 'object' && isInstanceOfSupportedElement(input.elt)) { + imgToPredict = input.elt; // Handle p5.js image and video. + } else if (typeof input === 'object' && isInstanceOfSupportedElement(input.canvas)) { + imgToPredict = input.canvas; // Handle p5.js image and video. + } + + if (imgToPredict) { + formattedInputs = imgToPixelArray(imgToPredict); + } else { + formattedInputs = input; + } + + return formattedInputs; } /** - * ---------------------------------------- - * ----- adding data / training ----------- - * ---------------------------------------- + * convertTrainingDataToTensors + * @param {*} _trainingData + * @param {*} _meta */ + convertTrainingDataToTensors(_trainingData = null, _meta = null) { + const trainingData = _trainingData === null ? this.data.training : _trainingData; + const meta = _meta === null ? this.neuralNetworkData.meta : _meta; + + return this.neuralNetworkData.convertRawToTensors(trainingData, meta); + } + /** - * Adds an endpoint to call data.addData() - * @param {*} xs - * @param {*} ys + * format the inputs for prediction + * this means applying onehot or normalization + * so that the user can use original data units rather + * than having to normalize + * @param {*} _input + * @param {*} meta + * @param {*} inputHeaders */ - addData(xs, ys) { - this.data.addData(xs, ys); + formatInputsForPrediction(_input, meta, inputHeaders) { + let inputData = []; + + // TODO: check to see if it is a nested array + // to run predict or classify on a batch of data + + if (_input instanceof Array) { + inputData = inputHeaders.map((prop, idx) => { + return this.isOneHotEncodedOrNormalized(_input[idx], prop, meta.inputs); + }); + } else if (_input instanceof Object) { + // TODO: make sure that the input order is preserved! + inputData = inputHeaders.map(prop => { + return this.isOneHotEncodedOrNormalized(_input[prop], prop, meta.inputs); + }); + } + + // inputData = tf.tensor([inputData.flat()]) + inputData = inputData.flat(); + + return inputData; } /** - * normalize the data.raw + * formatInputsForPredictionAll + * @param {*} _input + * @param {*} meta + * @param {*} inputHeaders */ - normalizeData() { - this.data.normalize(); + formatInputsForPredictionAll(_input, meta, inputHeaders) { + let output; + + if (_input instanceof Array) { + if (_input.every(item => Array.isArray(item))) { + output = _input.map(item => { + return this.formatInputsForPrediction(item, meta, inputHeaders); + }); + + return tf.tensor(output, [_input.length, inputHeaders.length]); + } + output = this.formatInputsForPrediction(_input, meta, inputHeaders); + return tf.tensor([output]); + } + + output = this.formatInputsForPrediction(_input, meta, inputHeaders); + return tf.tensor([output]); + } + + /** + * check if the input needs to be onehot encoded or + * normalized + * @param {*} _input + * @param {*} _meta + */ + // eslint-disable-next-line class-methods-use-this + isOneHotEncodedOrNormalized(_input, _key, _meta) { + const input = _input; + const key = _key; + + let output; + if (typeof _input !== 'number') { + output = _meta[key].legend[input]; + } else { + output = _input; + if (this.neuralNetworkData.meta.isNormalized) { + output = this.normalizeInput(_input, key, _meta); + } + } + return output; } + /** + * //////////////////////////////////////////////////////////// + * Model prep + * //////////////////////////////////////////////////////////// + */ /** - * User-facing neural network training + * train * @param {*} optionsOrCallback + * @param {*} optionsOrWhileTraining * @param {*} callback */ train(optionsOrCallback, optionsOrWhileTraining, callback) { let options; let whileTrainingCb; let finishedTrainingCb; - if (typeof optionsOrCallback === 'object' && + if ( + typeof optionsOrCallback === 'object' && typeof optionsOrWhileTraining === 'function' && typeof callback === 'function' ) { options = optionsOrCallback; whileTrainingCb = optionsOrWhileTraining; finishedTrainingCb = callback; - } else if (typeof optionsOrCallback === 'object' && - typeof optionsOrWhileTraining === 'function') { + } else if ( + typeof optionsOrCallback === 'object' && + typeof optionsOrWhileTraining === 'function' + ) { options = optionsOrCallback; whileTrainingCb = null; finishedTrainingCb = optionsOrWhileTraining; - } else if (typeof optionsOrCallback === 'function' && + } else if ( + typeof optionsOrCallback === 'function' && typeof optionsOrWhileTraining === 'function' ) { options = {}; @@ -310,643 +443,543 @@ class NeuralNetwork { finishedTrainingCb = optionsOrCallback; } - return callCallback(this.trainInternal(options, whileTrainingCb), finishedTrainingCb); + this.trainInternal(options, whileTrainingCb, finishedTrainingCb); } /** - * Train the neural network - * @param {*} options + * train + * @param {*} _options + * @param {*} _cb */ - async trainInternal(options, whileTrainingCallback) { - // get batch size and epochs - const batchSize = options.batchSize || this.config.training.batchSize; - const epochs = options.epochs || this.config.training.epochs; - - // placeholder for whiletraining callback; - let whileTraining; - // if debug is true, show tf vis during model training - // if not, then use whileTraining - let modelFitCallbacks; + trainInternal(_options, whileTrainingCb, finishedTrainingCb) { + const options = { + epochs: 10, + batchSize: 32, + validationSplit: 0.1, + whileTraining: null, + ..._options, + }; - // placeholder for xs and ys data for training - let xs; - let ys; + // if debug mode is true, then use tf vis + if (this.options.debug === true) { + options.whileTraining = [ + this.neuralNetworkVis.trainingVis(), + { + onEpochEnd: null, + }, + ]; + } else { + // if not use the default training + // options.whileTraining = whileTrainingCb === null ? [{ + // onEpochEnd: (epoch, loss) => { + // console.log(epoch, loss.loss) + // } + // }] : + // [{ + // onEpochEnd: whileTrainingCb + // }]; + options.whileTraining = [ + { + onEpochEnd: whileTrainingCb, + }, + ]; + } - // check if data are normalized, run the data.warmUp before training - if (!this.data.meta.isNormalized) { - this.data.warmUp(); + // if metadata needs to be generated about the data + if (!this.neuralNetworkData.isMetadataReady) { + // if the inputs are defined as an array of [img_width, img_height, channels] + this.createMetaData(this.neuralNetworkData.data.raw); } - // Get the inputs and outputs from the data object - // Ensure this comes AFTER .warmUp() in case - // .normalizeData() has not been called. - const { - inputs, - outputs - } = this.data.data.tensor; - - // Create the model when train is called - // important that this comes after checking if .isNormalized - this.model = this.createModel(); - - // check if a whileTrainingCallback was passed - if (typeof whileTrainingCallback === 'function') { - whileTraining = whileTrainingCallback; - } else { - whileTraining = () => null; + // if the data still need to be summarized, onehotencoded, etc + if (!this.neuralNetworkData.isWarmedUp) { + this.prepareForTraining(this.neuralNetworkData.data.raw); } - // check if the inputs are tensors, if not, convert! - if (!(inputs instanceof tf.Tensor)) { - xs = tf.tensor(inputs) - ys = tf.tensor(outputs) - } else { - xs = inputs; - ys = outputs; + // if inputs and outputs are not specified + // in the options, then create the tensors + // from the this.neuralNetworkData.data.raws + if (!options.inputs && !options.outputs) { + const { inputs, outputs } = this.convertTrainingDataToTensors(); + options.inputs = inputs; + options.outputs = outputs; } - // check if the debug mode is on to specify model fit callbacks - if (this.config.debug) { - modelFitCallbacks = [ - this.vis.trainingVis(), - { - onEpochEnd: whileTraining - } - ] - } else { - modelFitCallbacks = [{ - onEpochEnd: whileTraining - }] + // check to see if layers are passed into the constructor + // then use those to create your architecture + if (!this.neuralNetwork.isLayered) { + this.options.layers = this.createNetworkLayers( + this.options.layers, + this.neuralNetworkData.meta, + ); } - // train the model - await this.model.fit(xs, ys, { - shuffle: true, - batchSize, - epochs, - validationSplit: 0.1, - callbacks: modelFitCallbacks - }); - // dispose of the xs and ys - xs.dispose(); - ys.dispose(); - } + // if the model does not have any layers defined yet + // then use the default structure + if (!this.neuralNetwork.isLayered) { + this.options.layers = this.addDefaultLayers(this.options.task, this.neuralNetworkData.meta); + } + if (!this.neuralNetwork.isCompiled) { + // compile the model with defaults + this.compile(); + } - /** - * ---------------------------------------- - * ----- prediction / classification------- - * ---------------------------------------- - */ - /** - * Classify() - * Runs the classification if the neural network is doing a - * classification task - * @param {*} input - * @param {*} callback - */ - classify(input, callback) { - return callCallback(this.predictInternal(input), callback); + // train once the model is compiled + this.neuralNetwork.train(options, finishedTrainingCb); } /** - * classifyMultiple() - * Runs the classification task on multiple inputs - * @param {*} input - * @param {*} callback + * addLayer + * @param {*} options */ - classifyMultiple(input, callback) { - return callCallback(this.predictMultipleInternal(input), callback); + addLayer(options) { + this.neuralNetwork.addLayer(options); } /** - * User-facing prediction function - * @param {*} input - * @param {*} callback + * add custom layers in options */ - predict(input, callback) { - return callCallback(this.predictInternal(input), callback); + createNetworkLayers(layerJsonArray, meta) { + const layers = [...layerJsonArray]; + + const { inputUnits, outputUnits } = Object.assign({}, meta); + const layersLength = layers.length; + + if (!(layers.length >= 2)) { + return false; + } + + // set the inputShape + layers[0].inputShape = layers[0].inputShape ? layers[0].inputShape : inputUnits; + // set the output units + const lastIndex = layersLength - 1; + const lastLayer = layers[lastIndex]; + lastLayer.units = lastLayer.units ? lastLayer.units : outputUnits; + + layers.forEach(layer => { + this.addLayer(tf.layers[layer.type](layer)); + }); + + return layers; } + // /** + // * createDenseLayer + // * @param {*} _options + // */ + // // eslint-disable-next-line class-methods-use-this + // createDenseLayer(_options) { + // const options = Object.assign({}, { + // units: this.options.hiddenUnits, + // activation: 'relu', + // ..._options + // }); + // return tf.layers.dense(options); + // } + + // /** + // * createConv2dLayer + // * @param {*} _options + // */ + // // eslint-disable-next-line class-methods-use-this + // createConv2dLayer(_options) { + // const options = Object.assign({}, { + // kernelSize: 5, + // filters: 8, + // strides: 1, + // activation: 'relu', + // kernelInitializer: 'varianceScaling', + // ..._options + // }) + + // return tf.layers.conv2d(options); + // } + /** - * User-facing prediction function for multiple inputs - * @param {*} input - * @param {*} callback + * addDefaultLayers + * @param {*} _task */ - predictMultiple(input, callback) { - return callCallback(this.predictMultipleInternal(input), callback); + addDefaultLayers(task, meta) { + let layers; + switch (task.toLowerCase()) { + // if the task is classification + case 'classification': + layers = [ + { + type: 'dense', + units: this.options.hiddenUnits, + activation: 'relu', + }, + { + type: 'dense', + activation: 'softmax', + }, + ]; + + return this.createNetworkLayers(layers, meta); + // if the task is regression + case 'regression': + layers = [ + { + type: 'dense', + units: this.options.hiddenUnits, + activation: 'relu', + }, + { + type: 'dense', + activation: 'sigmoid', + }, + ]; + return this.createNetworkLayers(layers, meta); + // if the task is imageClassification + case 'imageclassification': + layers = [ + { + type: 'conv2d', + filters: 2, + kernelSize: 2, + strides: 2, + activation: 'relu', + kernelInitializer: 'varianceScaling', + }, + { + type: 'maxPooling2d', + poolSize: [1, 1], + strides: [1, 1], + }, + { + type: 'conv2d', + filters: 1, + kernelSize: 1, + strides: 1, + activation: 'relu', + kernelInitializer: 'varianceScaling', + }, + { + type: 'maxPooling2d', + poolSize: [1, 1], + strides: [1, 1], + }, + { + type: 'flatten', + }, + { + type: 'dense', + kernelInitializer: 'varianceScaling', + activation: 'softmax', + }, + ]; + return this.createNetworkLayers(layers, meta); + + default: + console.log('no imputUnits or outputUnits defined'); + layers = [ + { + type: 'dense', + units: this.options.hiddenUnits, + activation: 'relu', + }, + { + type: 'dense', + activation: 'sigmoid', + }, + ]; + return this.createNetworkLayers(layers, meta); + } } /** - * Make a prediction based on the given input - * @param {*} sample + * compile the model + * @param {*} _options */ - async predictMultipleInternal(sampleList) { + compile(_modelOptions = null, _learningRate = null) { + const LEARNING_RATE = _learningRate === null ? this.options.learningRate : _learningRate; - // 1. Handle the input sample - const inputData = []; - sampleList.forEach(sample => { - let inputRow = []; - if (sample instanceof Array) { - inputRow = sample; - } else if (sample instanceof Object) { - // TODO: make sure that the input order is preserved! - const headers = this.data.config.dataOptions.inputs; - inputRow = headers.map(prop => { - return sample[prop] - }); - } - inputData.push(inputRow); - }); + let options = {}; + if (_modelOptions !== null) { + options = { + ..._modelOptions, + }; + } else if ( + this.options.task === 'classification' || + this.options.task === 'imageClassification' + ) { + options = { + loss: 'categoricalCrossentropy', + optimizer: tf.train.sgd, + metrics: ['accuracy'], + }; + } else if (this.options.task === 'regression') { + options = { + loss: 'meanSquaredError', + optimizer: tf.train.adam, + metrics: ['accuracy'], + }; + } - // 2. onehot encode the sample if necessary - const encodedInput = []; + options.optimizer = options.optimizer + ? this.neuralNetwork.setOptimizerFunction(LEARNING_RATE, options.optimizer) + : this.neuralNetwork.setOptimizerFunction(LEARNING_RATE, tf.train.sgd); - sampleList.forEach(item => { - let encodedInputRow = []; - Object.entries(this.data.meta.inputs).forEach((arr) => { - const prop = arr[0]; - const { - dtype - } = arr[1]; + this.neuralNetwork.compile(options); - // to ensure that we get the value in the right order - const valIndex = this.data.config.dataOptions.inputs.indexOf(prop); - const val = item[valIndex]; + // if debug mode is true, then show the model summary + if (this.options.debug) { + this.neuralNetworkVis.modelSummary( + { + name: 'Model Summary', + }, + this.neuralNetwork.model, + ); + } + } - if (dtype === 'number') { - let normVal; - // if the data has not been normalized, just send in the raw sample - if (!this.data.meta.isNormalized) { - normVal = val; - } else { - const { - inputMin, - inputMax - } = this.data.data; - normVal = val; - if (inputMin && inputMax) { - normVal = (val - inputMin[valIndex]) / (inputMax[valIndex] - inputMin[valIndex]); - } - } - encodedInputRow.push(normVal); - } else if (dtype === 'string') { - const { - legend - } = arr[1]; - const onehotVal = legend[val] - encodedInputRow = [...encodedInputRow, ...onehotVal] - } - - encodedInput.push(encodedInputRow); - }); + /** + * //////////////////////////////////////////////////////////// + * Prediction / classification + * //////////////////////////////////////////////////////////// + */ - }); + /** + * predict + * @param {*} _input + * @param {*} _cb + */ + predict(_input, _cb) { + return callCallback(this.predictInternal(_input), _cb); + } + /** + * predictMultiple + * @param {*} _input + * @param {*} _cb + */ + predictMultiple(_input, _cb) { + return callCallback(this.predictInternal(_input), _cb); + } + /** + * classify + * @param {*} _input + * @param {*} _cb + */ + classify(_input, _cb) { + return callCallback(this.classifyInternal(_input), _cb); + } - // Step 3: make the prediction - const xs = tf.tensor(encodedInput, [encodedInput.length, this.data.meta.inputUnits]); - const ys = this.model.predict(xs); + /** + * classifyMultiple + * @param {*} _input + * @param {*} _cb + */ + classifyMultiple(_input, _cb) { + return callCallback(this.classifyInternal(_input), _cb); + } - // ys.print(); + /** + * predict + * @param {*} _input + * @param {*} _cb + */ + async predictInternal(_input) { + const { meta } = this.neuralNetworkData; + const headers = Object.keys(meta.inputs); - // Step 4: Convert the outputs back to the recognizable format - let results = []; + const inputData = this.formatInputsForPredictionAll(_input, meta, headers); - if (this.config.architecture.task === 'classification') { - const predictions = await ys.array(); - // TODO: Check to see if this fails with numeric values - // since no legend exists - const outputData = predictions.map(prediction => { - return Object.entries(this.data.meta.outputs).map((arr) => { - const { - legend - } = arr[1]; - // TODO: the order of the legend items matters - // Likey this means instead of `.push()`, - // we should do .unshift() - // alternatively we can use 'reverse()' here. - return Object.entries(legend).map((legendArr, idx) => { - const prop = legendArr[0]; - return { - label: prop, - confidence: prediction[idx] - } - }).sort((a, b) => b.confidence - a.confidence); - })[0]; - }) - // NOTE: we are doing a funky javascript thing - // setting an array as results, then adding - // .tensor as a property of that array object - results = outputData; - - // conditionally return the tensors if specified in options - if(this.config.returnTensors){ - results.tensor = ys; - } else { - results.tensor = null; - ys.dispose(); - } - - - } else if (this.config.architecture.task === 'regression') { - const predictions = await ys.array(); - - const outputData = predictions.map(prediction => { - return Object.entries(this.data.meta.outputs).map((item, idx) => { - const prop = item[0]; - const { - outputMin, - outputMax - } = this.data.data; + const unformattedResults = await this.neuralNetwork.predict(inputData); + inputData.dispose(); + + if (meta !== null) { + const labels = Object.keys(meta.outputs); + + const formattedResults = unformattedResults.map(unformattedResult => { + return labels.map((item, idx) => { + // check to see if the data were normalized + // if not, then send back the values, otherwise + // unnormalize then return let val; - if (!this.data.meta.isNormalized) { - val = prediction[idx] + let unNormalized; + if (meta.isNormalized) { + const { min, max } = meta.outputs[item]; + val = nnUtils.unnormalizeValue(unformattedResult[idx], min, max); + unNormalized = unformattedResult[idx]; } else { - val = (prediction[idx] * (outputMax[idx] - outputMin[idx])) + outputMin[idx]; + val = unformattedResult[idx]; } - return { + const d = { + [labels[idx]]: val, + label: item, value: val, - label: prop - } - }); - }) + }; + // if unNormalized is not undefined, then + // add that to the output + if (unNormalized) { + d.unNormalizedValue = unNormalized; + } - // NOTE: we are doing a funky javascript thing - // setting an array as results, then adding - // .tensor as a property of that array object - results = outputData; + return d; + }); + }); - // conditionally return the tensors if specified in options - if(this.config.returnTensors){ - results.tensor = ys; - } else { - results.tensor = null; - ys.dispose(); + // return single array if the length is less than 2, + // otherwise return array of arrays + if (formattedResults.length < 2) { + return formattedResults[0]; } + return formattedResults; } - xs.dispose(); - return results; - + // if no meta exists, then return unformatted results; + return unformattedResults; } /** - * Make a prediction based on the given input - * @param {*} sample + * classify + * @param {*} _input + * @param {*} _cb */ - async predictInternal(sample) { - // 1. Handle the input sample - // either an array of values in order of the inputs - // OR an JSON object of key/values - // console.log(sample) - - let inputData = []; - if (sample instanceof Array) { - inputData = sample; - } else if (sample instanceof Object) { - // TODO: make sure that the input order is preserved! - const headers = this.data.config.dataOptions.inputs; - inputData = headers.map(prop => { - return sample[prop] - }); - } - - - // 2. onehot encode the sample if necessary - let encodedInput = []; - - Object.entries(this.data.meta.inputs).forEach((arr) => { - const prop = arr[0]; - const { - dtype - } = arr[1]; - - // to ensure that we get the value in the right order - const valIndex = this.data.config.dataOptions.inputs.indexOf(prop); - const val = inputData[valIndex]; - - if (dtype === 'number') { - let normVal; - // if the data has not been normalized, just send in the raw sample - if (!this.data.meta.isNormalized) { - normVal = val; - } else { - const { - inputMin, - inputMax - } = this.data.data; - normVal = val; - if (inputMin && inputMax) { - normVal = (val - inputMin[valIndex]) / (inputMax[valIndex] - inputMin[valIndex]); - } - } - encodedInput.push(normVal); - } else if (dtype === 'string') { - const { - legend - } = arr[1]; - const onehotVal = legend[val] - encodedInput = [...encodedInput, ...onehotVal] - } - - }) - - const xs = tf.tensor(encodedInput, [1, this.data.meta.inputUnits]); - const ys = this.model.predict(xs); - - let results = []; - - if (this.config.architecture.task === 'classification') { - const predictions = await ys.data(); - // TODO: Check to see if this fails with numeric values - // since no legend exists - const outputData = Object.entries(this.data.meta.outputs).map((arr) => { - const { - legend - } = arr[1]; - // TODO: the order of the legend items matters - // Likey this means instead of `.push()`, - // we should do .unshift() - // alternatively we can use 'reverse()' here. - return Object.entries(legend).map((legendArr, idx) => { - const prop = legendArr[0]; - return { - label: prop, - confidence: predictions[idx] - } - }).sort((a, b) => b.confidence - a.confidence); - })[0]; - - // NOTE: we are doing a funky javascript thing - // setting an array as results, then adding - // .tensor as a property of that array object - results = outputData; - // conditionally return the tensors if specified in options - if(this.config.returnTensors){ - results.tensor = ys; + async classifyInternal(_input) { + const { meta } = this.neuralNetworkData; + const headers = Object.keys(meta.inputs); + + let inputData; + + if (this.options.task === 'imageClassification') { + // get the inputData for classification + // if it is a image type format it and + // flatten it + inputData = this.searchAndFormat(_input); + if (Array.isArray(inputData)) { + inputData = inputData.flat(); } else { - results.tensor = null; - ys.dispose(); + inputData = inputData[headers[0]]; } - } else if (this.config.architecture.task === 'regression') { - const predictions = await ys.data(); - - - const outputData = Object.entries(this.data.meta.outputs).map((item, idx) => { - const prop = item[0]; - const { - outputMin, - outputMax - } = this.data.data; - let val; - if (!this.data.meta.isNormalized) { - val = predictions[idx] - } else { - val = (predictions[idx] * (outputMax[idx] - outputMin[idx])) + outputMin[idx]; - } - - return { - value: val, - label: prop - } - }); - - - // NOTE: we are doing a funky javascript thing - // setting an array as results, then adding - // .tensor as a property of that array object - results = outputData; - // conditionally return the tensors if specified in options - if(this.config.returnTensors){ - results.tensor = ys; + if (meta.isNormalized) { + // TODO: check to make sure this property is not static!!!! + const { min, max } = meta.inputs[headers[0]]; + inputData = this.neuralNetworkData.normalizeArray(Array.from(inputData), { min, max }); } else { - results.tensor = null; - ys.dispose(); + inputData = Array.from(inputData); } + + inputData = tf.tensor([inputData], [1, ...meta.inputUnits]); + } else { + inputData = this.formatInputsForPredictionAll(_input, meta, headers); } - xs.dispose(); - return results; + const unformattedResults = await this.neuralNetwork.classify(inputData); + inputData.dispose(); + if (meta !== null) { + const label = Object.keys(meta.outputs)[0]; + const vals = Object.entries(meta.outputs[label].legend); - } - + const formattedResults = unformattedResults.map(unformattedResult => { + return vals + .map((item, idx) => { + return { + [item[0]]: unformattedResult[idx], + label: item[0], + confidence: unformattedResult[idx], + }; + }) + .sort((a, b) => b.confidence - a.confidence); + }); + // return single array if the length is less than 2, + // otherwise return array of arrays + if (formattedResults.length < 2) { + return formattedResults[0]; + } + return formattedResults; + } + return unformattedResults; + } /** - * ---------------------------------------- - * ----- Exporting / Saving --------------- - * ---------------------------------------- + * //////////////////////////////////////////////////////////// + * Save / Load Data + * //////////////////////////////////////////////////////////// */ /** - * Calls this.data.saveData() to save data out to a json file - * @param {*} callback + * save data * @param {*} name */ - async saveData(nameOrCallback, callback) { - let cb; - let outputName; - - // check the inputs - if (typeof nameOrCallback === 'string' && callback) { - outputName = nameOrCallback - cb = callback; - } else if (typeof nameOrCallback === 'string' && !callback) { - cb = null; - outputName = nameOrCallback - } else if (typeof nameOrCallback === 'function') { - cb = nameOrCallback - outputName = undefined; - } - - // save the data out - await this.data.saveData(outputName); - - if (typeof cb === 'function') { - cb(); - } + saveData(name) { + this.neuralNetworkData.saveData(name); } /** - * loadData from fileinput or path + * load data * @param {*} filesOrPath * @param {*} callback */ async loadData(filesOrPath = null, callback) { - - let loadedData; - if (typeof filesOrPath !== 'string') { - const file = filesOrPath[0]; - const fr = new FileReader(); - fr.readAsText(file); - if (file.name.includes('.json')) { - const temp = await file.text(); - loadedData = JSON.parse(temp); - } else { - console.log('data must be a json object containing an array called "data" or "entries') - } - } else { - loadedData = await fetch(filesOrPath); - const text = await loadedData.text(); - if (this.data.isJsonString(text)) { - loadedData = JSON.parse(text); - } else { - console.log('Whoops! something went wrong. Either this kind of data is not supported yet or there is an issue with .loadData') - } - } - - // check if a data or entries property exists - if (loadedData.data) { - this.data.data.raw = loadedData.data; - } else if (loadedData.entries) { - this.data.data.raw = loadedData.entries; - } else { - console.log('data must be a json object containing an array called "data" or "entries') - } - - if (callback) { - callback(); - } + this.neuralNetworkData.loadData(filesOrPath, callback); } - /** - * Save the model and weights - * @param {*} callback - * @param {*} name + * //////////////////////////////////////////////////////////// + * Save / Load Model + * //////////////////////////////////////////////////////////// */ - async save(callback, name) { - this.model.save(tf.io.withSaveHandler(async (data) => { - let modelName = 'model'; - if (name) modelName = name; - - this.weightsManifest = { - modelTopology: data.modelTopology, - weightsManifest: [{ - paths: [`./${modelName}.weights.bin`], - weights: data.weightSpecs, - }] - }; - const dataMeta = { - data: { - inputMin: this.data.data.inputMin, - inputMax: this.data.data.inputMax, - outputMin: this.data.data.outputMin, - outputMax: this.data.data.outputMax, - }, - meta: this.data.meta + /** + * saves the model, weights, and metadata + * @param {*} nameOrCb + * @param {*} cb + */ + save(nameOrCb, cb) { + let modelName; + let callback; + + if (typeof nameOrCb === 'function') { + modelName = 'model'; + callback = nameOrCb; + } else if (typeof nameOrCb === 'string') { + modelName = nameOrCb; + + if (typeof cb === 'function') { + callback = cb; } + } else { + modelName = 'model'; + } - await saveBlob(data.weightData, `${modelName}.weights.bin`, 'application/octet-stream'); - await saveBlob(JSON.stringify(this.weightsManifest), `${modelName}.json`, 'text/plain'); - await saveBlob(JSON.stringify(dataMeta), `${modelName}_meta.json`, 'text/plain'); - if (callback) { - callback(); - } - })); + // save the model + this.neuralNetwork.save(modelName, () => { + this.neuralNetworkData.saveMeta(modelName, callback); + }); } /** - * Load the model and weights in from a file + * load a model and metadata * @param {*} filesOrPath * @param {*} callback */ - async load(filesOrPath = null, callback) { - - if (filesOrPath instanceof FileList) { - - const files = await Promise.all( - Array.from(filesOrPath).map( async (file) => { - if (file.name.includes('model.json')) { - return {name:"model", file} - } else if (file.name.includes('_meta.json')) { - const modelMetadata = await file.text(); - return {name: "metadata", file:modelMetadata} - } else if (file.name.includes('.bin')) { - return {name:"weights", file} - } - return {name:null, file:null} - }) - ) - - const model = files.find(item => item.name === 'model').file; - const modelMetadata = JSON.parse(files.find(item => item.name === 'metadata').file); - const weights = files.find(item => item.name === 'weights').file; - - // set the metainfo - this.data.data.inputMax = modelMetadata.data.inputMax; - this.data.data.inputMin = modelMetadata.data.inputMin; - this.data.data.outputMax = modelMetadata.data.outputMax; - this.data.data.outputMin = modelMetadata.data.outputMin; - this.data.meta = modelMetadata.meta; - - // load the model - this.model = await tf.loadLayersModel(tf.io.browserFiles([model, weights])); - - } else if(filesOrPath instanceof Object){ - // filesOrPath = {model: URL, metadata: URL, weights: URL} - - let modelMetadata = await fetch(filesOrPath.metadata); - modelMetadata = await modelMetadata.text(); - modelMetadata = JSON.parse(modelMetadata); - - let modelJson = await fetch(filesOrPath.model); - modelJson = await modelJson.text(); - const modelJsonFile = new File([modelJson], 'model.json', {type: 'application/json'}); - - let weightsBlob = await fetch(filesOrPath.weights); - weightsBlob = await weightsBlob.blob(); - const weightsBlobFile = new File([weightsBlob], 'model.weights.bin', {type: 'application/macbinary'}); - - this.data.data.inputMax = modelMetadata.data.inputMax; - this.data.data.inputMin = modelMetadata.data.inputMin; - this.data.data.outputMax = modelMetadata.data.outputMax; - this.data.data.outputMin = modelMetadata.data.outputMin; - this.data.meta = modelMetadata.meta; - - this.model = await tf.loadLayersModel(tf.io.browserFiles([modelJsonFile, weightsBlobFile])); - - } else { - const metaPath = `${filesOrPath.substring(0, filesOrPath.lastIndexOf("/"))}/model_meta.json`; - let modelMetadata = await fetch(metaPath); - modelMetadata = await modelMetadata.json(); + async load(filesOrPath = null, cb) { + let callback; + if (cb) { + callback = cb; + } - this.data.data.inputMax = modelMetadata.data.inputMax; - this.data.data.inputMin = modelMetadata.data.inputMin; - this.data.data.outputMax = modelMetadata.data.outputMax; - this.data.data.outputMin = modelMetadata.data.outputMin; - this.data.meta = modelMetadata.meta; + this.neuralNetwork.load(filesOrPath, () => { + this.neuralNetworkData.loadMeta(filesOrPath, callback); - this.model = await tf.loadLayersModel(filesOrPath); - } - if (callback) { - callback(); - } - return this.model; + return this.neuralNetwork.model; + }); } - } - - - -/** - * Create an instance of the NeuralNetwork - * @param {*} inputsOrOptions - * @param {*} outputsOrCallback - * @param {*} callback - */ const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { - let options; let cb; @@ -961,8 +994,8 @@ const neuralNetwork = (inputsOrOptions, outputsOrCallback, callback) => { cb = callback; } - const instance = new NeuralNetwork(options, cb); + const instance = new DiyNeuralNetwork(options, cb); return instance; }; -export default neuralNetwork; \ No newline at end of file +export default neuralNetwork; diff --git a/src/NeuralNetwork/index_test.js b/src/NeuralNetwork/index_test.js index ca09f47ce..e69de29bb 100644 --- a/src/NeuralNetwork/index_test.js +++ b/src/NeuralNetwork/index_test.js @@ -1,63 +0,0 @@ -// Copyright (c) 2019 ml5 -// -// This software is released under the MIT License. -// https://opensource.org/licenses/MIT - - -const { - neuralNetwork -} = ml5; - -const NN_DEFAULTS = { - task: 'regression', - activationHidden: 'sigmoid', - activationOutput: 'sigmoid', - debug: false, - learningRate: 0.25, - inputs: 2, - outputs: 1, - noVal: null, - hiddenUnits: 16, - modelMetrics: ['accuracy'], - modelLoss: 'meanSquaredError', - modelOptimizer: null, - batchSize: 64, - epochs: 32, -} - - -describe('neuralNetwork', () => { - let nn; - - beforeAll(async () => { - jasmine.DEFAULT_TIMEOUT_INTERVAL = 15000; - nn = await neuralNetwork(); - }); - - it('Should create neuralNetwork with all the defaults', async () => { - expect(nn.config.debug).toBe(NN_DEFAULTS.debug); - // architecture defaults - expect(nn.config.architecture.task).toBe(NN_DEFAULTS.task); - - // expect(nn.config.architecture.layers).toBe(); - // expect(nn.config.architecture.activations).toBe(NN_DEFAULTS.activations); - - // training defaults - expect(nn.config.training.batchSize).toBe(NN_DEFAULTS.batchSize); - expect(nn.config.training.epochs).toBe(NN_DEFAULTS.epochs); - expect(nn.config.training.learningRate).toBe(NN_DEFAULTS.learningRate); - - // expect(nn.config.training.modelMetrics).toBe(NN_DEFAULTS.modelMetrics); - expect(nn.config.training.modelLoss).toBe(NN_DEFAULTS.modelLoss); - // expect(nn.config.training.modelOptimizer).toBe(); - - // data defaults - // expect(nn.config.dataOptions.dataUrl).toBe(); - // expect(nn.config.dataOptions.inputs).toBe(NN_DEFAULTS.inputs); - // expect(nn.config.dataOptions.outputs).toBe(NN_DEFAULTS.outputs); - - // expect(nn.config.dataOptions.normalizationOptions).toBe(); - - }); - -}); \ No newline at end of file diff --git a/src/index.js b/src/index.js index 8c973145b..9be1e57c7 100644 --- a/src/index.js +++ b/src/index.js @@ -4,6 +4,7 @@ // https://opensource.org/licenses/MIT import * as tf from '@tensorflow/tfjs'; +import * as tfvis from '@tensorflow/tfjs-vis'; import pitchDetection from './PitchDetection/'; import imageClassifier from './ImageClassifier/'; import soundClassifier from './SoundClassifier/'; @@ -56,6 +57,7 @@ module.exports = Object.assign({p5Utils}, preloadRegister(withPreload), { KNNClassifier, ...imageUtils, tf, + tfvis, version, neuralNetwork, }); diff --git a/src/utils/imageUtilities.js b/src/utils/imageUtilities.js index 902252c0e..812295c63 100644 --- a/src/utils/imageUtilities.js +++ b/src/utils/imageUtilities.js @@ -141,11 +141,57 @@ function isInstanceOfSupportedElement(subject) { || subject instanceof ImageData) } +function imgToPixelArray(img){ + // image image, bitmap, or canvas + let imgWidth; + let imgHeight; + let inputImg; + + if (img instanceof HTMLImageElement || + img instanceof HTMLCanvasElement || + img instanceof HTMLVideoElement || + img instanceof ImageData) { + inputImg = img; + } else if (typeof img === 'object' && + (img.elt instanceof HTMLImageElement || + img.elt instanceof HTMLCanvasElement || + img.elt instanceof HTMLVideoElement || + img.elt instanceof ImageData)) { + + inputImg = img.elt; // Handle p5.js image + } else if (typeof img === 'object' && + img.canvas instanceof HTMLCanvasElement) { + inputImg = img.canvas; // Handle p5.js image + } else { + inputImg = img; + } + + if (inputImg instanceof HTMLVideoElement) { + // should be videoWidth, videoHeight? + imgWidth = inputImg.width; + imgHeight = inputImg.height; + } else { + imgWidth = inputImg.width; + imgHeight = inputImg.height; + } + + const canvas = document.createElement('canvas'); + canvas.width = imgWidth; + canvas.height = imgHeight; + + const ctx = canvas.getContext('2d'); + ctx.drawImage(inputImg, 0, 0, imgWidth, imgHeight); + + const imgData = ctx.getImageData(0,0, imgWidth, imgHeight) + return Array.from(imgData.data) +} + export { array3DToImage, processVideo, cropImage, imgToTensor, isInstanceOfSupportedElement, - flipImage + flipImage, + imgToPixelArray };