Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix json parse bug in neuralNetworkData loadData #207

Merged
merged 1 commit into from
Sep 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/NeuralNetwork/NeuralNetwork.js
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,9 @@ class NeuralNetwork {
if (filesOrPath instanceof FileList) {
const files = Array.from(filesOrPath);
// find the correct files
const model = files.find((file) => file.name.includes(".json") && !file.name.includes("_meta"));
const model = files.find(
(file) => file.name.includes(".json") && !file.name.includes("_meta")
);
const weights = files.find((file) => file.name.includes(".bin"));
// load the model
this.model = await tf.loadLayersModel(
Expand All @@ -218,7 +220,7 @@ class NeuralNetwork {
// Override the weights path from the JSON weightsManifest
weightUrlConverter: (weightFileName) => {
return filesOrPath.weights || weightFileName;
}
},
})
);
} else {
Expand Down
23 changes: 10 additions & 13 deletions src/NeuralNetwork/NeuralNetworkData.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as tf from "@tensorflow/tfjs";
import axios from "axios";
import { saveBlob } from "../utils/io";
import modelLoader from '../utils/modelLoader';
import modelLoader from "../utils/modelLoader";
import nnUtils from "./NeuralNetworkUtils";

class NeuralNetworkData {
Expand Down Expand Up @@ -532,7 +532,6 @@ class NeuralNetworkData {
*/
async loadDataFromUrl(dataUrl, inputs, outputs) {
try {

if (dataUrl.endsWith(".csv")) {
await this.loadCSV(dataUrl, inputs, outputs);
} else if (dataUrl.endsWith(".json")) {
Expand Down Expand Up @@ -643,23 +642,21 @@ class NeuralNetworkData {
);
}
} else {
loadedData = await axios.get(filesOrPath, { responseType: "text" });
const text = JSON.stringify(loadedData.data);
if (nnUtils.isJsonOrString(text)) {
loadedData = JSON.parse(text);
let response = await axios.get(filesOrPath, { responseType: "text" });

if (nnUtils.isJsonOrString(response.data)) {
loadedData = JSON.parse(response.data);
} else {
console.log(
"Whoops! something went wrong. Either this kind of data is not supported yet or there is an issue with .loadData"
console.error(
"🟪 ml5.js error: `NeuralNetwork.loadData` only accepts JSON data."
);
}
}

this.data.raw = this.findEntries(loadedData);

// check if a data or entries property exists
if (!this.data.raw.length > 0) {
console.log(
'data must be a json object containing an array called "data" '
console.error(
"🟪 ml5.js error: `NeuralNetwork.loadData` only accepts JSON objects with a 'data' property."
);
}
} catch (error) {
Expand Down Expand Up @@ -716,7 +713,7 @@ class NeuralNetworkData {
file.name.includes("_meta.json")
);
if (!file) {
console.warn('no model_meta.json file found in FileList');
console.warn("no model_meta.json file found in FileList");
return;
}
const text = await file.text();
Expand Down
50 changes: 29 additions & 21 deletions src/NeuralNetwork/index.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import * as tf from "@tensorflow/tfjs";
import callCallback from "../utils/callcallback";
import handleArguments from "../utils/handleArguments";
import { imgToPixelArray, isInstanceOfSupportedElement, } from "../utils/imageUtilities";
import {
imgToPixelArray,
isInstanceOfSupportedElement,
} from "../utils/imageUtilities";
import NeuralNetwork from "./NeuralNetwork";
import NeuralNetworkData from "./NeuralNetworkData";

Expand All @@ -22,7 +25,6 @@ const DEFAULTS = {
};
class DiyNeuralNetwork {
constructor(options, callback) {

// Is there a better way to handle a different
// default learning rate for image classification tasks?
if (options.task === "imageClassification") {
Expand Down Expand Up @@ -227,11 +229,7 @@ class DiyNeuralNetwork {
async loadDataFromUrl() {
const { dataUrl, inputs, outputs } = this.options;

await this.neuralNetworkData.loadDataFromUrl(
dataUrl,
inputs,
outputs
);
await this.neuralNetworkData.loadDataFromUrl(dataUrl, inputs, outputs);

// once the data are loaded, create the metadata
// and prep the data for training
Expand Down Expand Up @@ -512,7 +510,10 @@ class DiyNeuralNetwork {
finishedTrainingCb = optionsOrCallback;
}

return callCallback(this.trainInternal(options, whileTrainingCb), finishedTrainingCb);
return callCallback(
this.trainInternal(options, whileTrainingCb),
finishedTrainingCb
);
}

/**
Expand Down Expand Up @@ -579,9 +580,7 @@ class DiyNeuralNetwork {
// then use those to create your architecture
if (!this.neuralNetwork.isLayered) {
// TODO: don't update this.options.layers - Linda
this.options.layers = this.createNetworkLayers(
this.options.layers
);
this.options.layers = this.createNetworkLayers(this.options.layers);
}

// if the model does not have any layers defined yet
Expand Down Expand Up @@ -1150,7 +1149,10 @@ class DiyNeuralNetwork {
*/
saveData(name, callback) {
const args = handleArguments(name, callback);
return callCallback(this.neuralNetworkData.saveData(args.name), args.callback);
return callCallback(
this.neuralNetworkData.saveData(args.name),
args.callback
);
}

/**
Expand Down Expand Up @@ -1182,13 +1184,16 @@ class DiyNeuralNetwork {
*/
async save(name, callback) {
const args = handleArguments(name, callback);
const modelName = args.string || 'model';
const modelName = args.string || "model";

// save the model
return callCallback(Promise.all([
this.neuralNetwork.save(modelName),
this.neuralNetworkData.saveMeta(modelName)
]), args.callback);
return callCallback(
Promise.all([
this.neuralNetwork.save(modelName),
this.neuralNetworkData.saveMeta(modelName),
]),
args.callback
);
}

/**
Expand All @@ -1200,10 +1205,13 @@ class DiyNeuralNetwork {
* @return {Promise<void[]>}
*/
async load(filesOrPath, callback) {
return callCallback(Promise.all([
this.neuralNetwork.load(filesOrPath),
this.neuralNetworkData.loadMeta(filesOrPath)
]), callback);
return callCallback(
Promise.all([
this.neuralNetwork.load(filesOrPath),
this.neuralNetworkData.loadMeta(filesOrPath),
]),
callback
);
}

/**
Expand Down