From 8805ad13f486cb58eaf26037a3cd5320a51e25ab Mon Sep 17 00:00:00 2001 From: shiffman Date: Sun, 29 Sep 2019 18:02:30 -0400 Subject: [PATCH 1/6] neural noteplayer example --- .../NeuralNetwork_musical_notes/index.html | 31 ++++++++++ .../NeuralNetwork_musical_notes/sketch.js | 62 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100755 p5js/NeuralNetwork/NeuralNetwork_musical_notes/index.html create mode 100644 p5js/NeuralNetwork/NeuralNetwork_musical_notes/sketch.js diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_notes/index.html b/p5js/NeuralNetwork/NeuralNetwork_musical_notes/index.html new file mode 100755 index 00000000..1e3de00a --- /dev/null +++ b/p5js/NeuralNetwork/NeuralNetwork_musical_notes/index.html @@ -0,0 +1,31 @@ + + + + + Neural Network Sound Player + + + + + + + + +

Neural Network Sound Player

+ + +

+ Training frequency: + +

+

+ Frequency prediction: +

+ +

+ Click in canvas to add training data. +

+ + + + \ No newline at end of file diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_notes/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_musical_notes/sketch.js new file mode 100644 index 00000000..c3be7c68 --- /dev/null +++ b/p5js/NeuralNetwork/NeuralNetwork_musical_notes/sketch.js @@ -0,0 +1,62 @@ +let notePlayer; +let playing = false; +let frequency; +let osc; + +function setup() { + createCanvas(400, 400).mousePressed(addData);; + const options = { + inputs: 2, // what about allowing ['x', 'y']? + outputs: 1, // what about allowing ['x', 'y']? + debug: true, + } + background(0); + notePlayer = ml5.neuralNetwork(options); + select('#train').mousePressed(trainModel); +} + +function addData() { + + let freq = select('#frequency').value(); + stroke(255); + noFill(); + ellipse(mouseX, mouseY, 32); + fill(255); + textSize(16); + console.log(freq); + textAlign(CENTER, CENTER); + text(freq, mouseX, mouseY); + notePlayer.data.addData([mouseX, mouseY], [parseFloat(freq)]); +} + +function trainModel() { + notePlayer.data.normalize(); + const trainingOptions = { + batchSize: 24, + epochs: 20 + } + notePlayer.train(trainingOptions, finishedTraining); +} + +function finishedTraining() { + console.log('done'); + osc = new p5.Oscillator(); + osc.setType('sine'); + osc.amp(0.5); + osc.freq(440); + osc.start(); + notePlayer.predict([mouseX, mouseY], gotFrequency); +} + +function gotFrequency(error, results) { + if (error) { + console.error(error); + } else { + frequency = parseFloat(results.outputs.value); + console.log(results); + select('#prediction').html(frequency.toFixed(2)); + osc.freq(parseFloat(frequency)); + notePlayer.predict([mouseX, mouseY], gotFrequency); + } +} + From ac58f36c0e20715e8a390a55b6849ab0766f1e43 Mon Sep 17 00:00:00 2001 From: shiffman Date: Mon, 30 Sep 2019 22:41:00 -0400 Subject: [PATCH 2/6] example inspired by Eyeo 2019 Rebecca Fiebrink talk --- .../NeuralNetwork_lowres_pixels/index.html | 27 +++++ .../NeuralNetwork_lowres_pixels/sketch.js | 102 ++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100755 p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/index.html create mode 100644 p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js diff --git a/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/index.html b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/index.html new file mode 100755 index 00000000..c76027ae --- /dev/null +++ b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/index.html @@ -0,0 +1,27 @@ + + + + + Neural Network Sound Player + + + + + + + + +

Pixel Prediction

+ + +

+ Training frequency: + +

+

+ + Frequency prediction: +

+ + + \ No newline at end of file diff --git a/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js new file mode 100644 index 00000000..b0d9f445 --- /dev/null +++ b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js @@ -0,0 +1,102 @@ + + +let pixelBrain; +let video; + + +let ready = false; +let w; + +function setup() { + createCanvas(200, 200); + video = createCapture(VIDEO, videoReady); + let res = 10; + video.size(res, res); + video.hide(); + let totalPixels = res * res * 3; + const options = { + inputs: totalPixels, + outputs: 1, + hiddenUnits: floor(totalPixels / 2), + activationHidden: 'relu', + learningRate: 0.01, + debug: true, + } + pixelBrain = ml5.neuralNetwork(options); + select('#addExample').mousePressed(addExample); + select('#train').mousePressed(trainModel); + w = width / res; +} + +function videoReady() { + ready = true; +} + +function draw() { + background(0); + if (ready) { + video.loadPixels(); + for (let x = 0; x < video.width; x++) { + for (let y = 0; y < video.height; y++) { + let index = (x + y * video.width) * 4; + let r = video.pixels[index + 0]; + let g = video.pixels[index + 1]; + let b = video.pixels[index + 2]; + noStroke(); + fill(r, g, b); + rect(x * w, y * w, w, w); + } + } + } + +} + +function getInputs() { + video.loadPixels(); + let inputs = []; + for (let i = 0; i < video.width * video.height; i++) { + let index = i * 4; + inputs.push(video.pixels[index + 0]); + inputs.push(video.pixels[index + 1]); + inputs.push(video.pixels[index + 2]); + } + return inputs; +} + +function addExample() { + let freq = select('#frequency').value(); + video.loadPixels(); + let inputs = getInputs(); + pixelBrain.data.addData(inputs, [parseFloat(freq)]); +} + +function trainModel() { + pixelBrain.data.normalize(); + const trainingOptions = { + epochs: 100 + } + pixelBrain.train(trainingOptions, finishedTraining); +} + +function finishedTraining() { + console.log('done'); + predict(); +} + +function predict() { + let inputs = getInputs(); + pixelBrain.predict(inputs, gotFrequency); +} + +function gotFrequency(error, results) { + if (error) { + console.error(error); + } else { + frequency = parseFloat(results.outputs.value); + console.log(results); + select('#prediction').html(frequency.toFixed(2)); + // osc.freq(parseFloat(frequency)); + predict(); + } +} + From a6771124ec918e031adadb41681e7dd29a9aca0d Mon Sep 17 00:00:00 2001 From: shiffman Date: Mon, 30 Sep 2019 22:42:58 -0400 Subject: [PATCH 3/6] rename musical mouse example --- .../index.html | 0 .../sketch.js | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename p5js/NeuralNetwork/{NeuralNetwork_musical_notes => NeuralNetwork_musical_mouse}/index.html (100%) rename p5js/NeuralNetwork/{NeuralNetwork_musical_notes => NeuralNetwork_musical_mouse}/sketch.js (100%) diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_notes/index.html b/p5js/NeuralNetwork/NeuralNetwork_musical_mouse/index.html similarity index 100% rename from p5js/NeuralNetwork/NeuralNetwork_musical_notes/index.html rename to p5js/NeuralNetwork/NeuralNetwork_musical_mouse/index.html diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_notes/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_musical_mouse/sketch.js similarity index 100% rename from p5js/NeuralNetwork/NeuralNetwork_musical_notes/sketch.js rename to p5js/NeuralNetwork/NeuralNetwork_musical_mouse/sketch.js From c08c9aee5cd1ae43c2f9ec3e5fec4d0f0204a9a2 Mon Sep 17 00:00:00 2001 From: shiffman Date: Mon, 30 Sep 2019 23:16:38 -0400 Subject: [PATCH 4/6] example tweaks and oscillator --- .../NeuralNetwork_lowres_pixels/sketch.js | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js index b0d9f445..d31d0545 100644 --- a/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js +++ b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js @@ -2,10 +2,11 @@ let pixelBrain; let video; - - let ready = false; let w; +let playing = false; +let frequency; +let osc; function setup() { createCanvas(200, 200); @@ -18,7 +19,7 @@ function setup() { inputs: totalPixels, outputs: 1, hiddenUnits: floor(totalPixels / 2), - activationHidden: 'relu', + // activationHidden: 'relu', learningRate: 0.01, debug: true, } @@ -26,6 +27,10 @@ function setup() { select('#addExample').mousePressed(addExample); select('#train').mousePressed(trainModel); w = width / res; + osc = new p5.Oscillator(); + osc.setType('sine'); + osc.amp(0.5); + osc.freq(440); } function videoReady() { @@ -63,23 +68,32 @@ function getInputs() { return inputs; } +let firstTime = true; function addExample() { + if (firstTime) { + osc.start(); + firstTime = false; + } + let freq = select('#frequency').value(); + osc.freq(parseFloat(freq)); video.loadPixels(); let inputs = getInputs(); pixelBrain.data.addData(inputs, [parseFloat(freq)]); } function trainModel() { + osc.amp(0); pixelBrain.data.normalize(); const trainingOptions = { - epochs: 100 + epochs: 50 } pixelBrain.train(trainingOptions, finishedTraining); } function finishedTraining() { console.log('done'); + osc.amp(0.5); predict(); } @@ -93,9 +107,8 @@ function gotFrequency(error, results) { console.error(error); } else { frequency = parseFloat(results.outputs.value); - console.log(results); select('#prediction').html(frequency.toFixed(2)); - // osc.freq(parseFloat(frequency)); + osc.freq(parseFloat(frequency)); predict(); } } From 8ec1259aa6f64285e2e6c9bac8e8066db32655c1 Mon Sep 17 00:00:00 2001 From: shiffman Date: Tue, 1 Oct 2019 14:57:36 -0400 Subject: [PATCH 5/6] musical face example --- .../NeuralNetwork_musical_face/index.html | 28 ++++ .../NeuralNetwork_musical_face/sketch.js | 125 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100755 p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html create mode 100644 p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html b/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html new file mode 100755 index 00000000..45c383f1 --- /dev/null +++ b/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html @@ -0,0 +1,28 @@ + + + + + Neural Network Sound Player + + + + + + + + +

Face Prediction

+ + +

+ Training frequency: 440 + + +

+

+ + Frequency prediction: +

+ + + \ No newline at end of file diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js new file mode 100644 index 00000000..a4fe04d3 --- /dev/null +++ b/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js @@ -0,0 +1,125 @@ +let faceapi; +let video; +let detections; + +let faceBrain; +let osc; +let trained = false; +let playing = false; + + +function setup() { + createCanvas(360, 270); + video = createCapture(VIDEO); + video.size(width, height); + // video.hide(); + + const faceOptions = { + withLandmarks: true, + withExpressions: false, + withDescriptors: false, + } + faceapi = ml5.faceApi(video, faceOptions, modelReady); + + const options = { + inputs: 68 * 2, + outputs: 1, + hiddenUnits: 68, + learningRate: 0.01, + debug: true, + } + faceBrain = ml5.neuralNetwork(options); + select('#addExample').mousePressed(addExample); + select('#train').mousePressed(trainModel); + + osc = new p5.Oscillator(); + osc.setType('sine'); + osc.amp(0.5); + select('#frequency_slider').input(function () { + if (!playing) { + osc.start(); + playing = true; + } + let freq = select('#frequency_slider').value(); + select('#training_freq').html(freq); + if (!trained) { + osc.freq(freq); + } + }) +} + +function modelReady() { + faceapi.detect(gotResults); +} + +function gotResults(err, result) { + if (err) { + console.log(err); + return; + } + detections = result; + faceapi.detect(gotResults); +} + +function draw() { + background(0); + if (detections && detections.length > 0) { + let points = detections[0].landmarks.positions; + for (let i = 0; i < points.length; i++) { + stroke(161, 95, 251); + strokeWeight(4); + point(points[i]._x, points[i]._y); + } + } +} + +function getInputs() { + if (detections && detections.length > 0) { + let points = detections[0].landmarks.positions; + let inputs = []; + for (let i = 0; i < points.length; i++) { + inputs.push(points[i]._x); + inputs.push(points[i]._y); + } + return inputs; + } +} + +function addExample() { + let freq = select('#frequency_slider').value(); + let inputs = getInputs(); + if (inputs) { + faceBrain.data.addData(inputs, [parseFloat(freq)]); + } +} + +function trainModel() { + osc.amp(0); + faceBrain.data.normalize(); + const trainingOptions = { + epochs: 50 + } + faceBrain.train(trainingOptions, finishedTraining); +} + +function finishedTraining() { + console.log('done'); + osc.amp(0.5); + predict(); +} + +function predict() { + let inputs = getInputs(); + faceBrain.predict(inputs, gotFrequency); +} + +function gotFrequency(error, results) { + if (error) { + console.error(error); + } else { + frequency = parseFloat(results.outputs.value); + osc.freq(frequency); + select('#prediction').html(frequency.toFixed(2)); + predict(); + } +} From 55ccb54b46802b242071b9f1bd36b7018604a170 Mon Sep 17 00:00:00 2001 From: shiffman Date: Tue, 1 Oct 2019 15:23:06 -0400 Subject: [PATCH 6/6] different interface for face example --- .../NeuralNetwork_musical_face/index.html | 2 +- .../NeuralNetwork_musical_face/sketch.js | 36 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html b/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html index 45c383f1..9ecc0c04 100755 --- a/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html +++ b/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html @@ -17,7 +17,7 @@

Face Prediction

Training frequency: 440 - +

diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js index a4fe04d3..74ac3671 100644 --- a/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js +++ b/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js @@ -5,7 +5,7 @@ let detections; let faceBrain; let osc; let trained = false; -let playing = false; +let collecting = false; function setup() { @@ -29,23 +29,11 @@ function setup() { debug: true, } faceBrain = ml5.neuralNetwork(options); - select('#addExample').mousePressed(addExample); + select('#collectData').mousePressed(collectData); select('#train').mousePressed(trainModel); osc = new p5.Oscillator(); osc.setType('sine'); - osc.amp(0.5); - select('#frequency_slider').input(function () { - if (!playing) { - osc.start(); - playing = true; - } - let freq = select('#frequency_slider').value(); - select('#training_freq').html(freq); - if (!trained) { - osc.freq(freq); - } - }) } function modelReady() { @@ -71,6 +59,15 @@ function draw() { point(points[i]._x, points[i]._y); } } + if (collecting) { + let freq = select('#frequency_slider').value(); + select('#training_freq').html(freq); + osc.freq(freq); + let inputs = getInputs(); + if (inputs) { + faceBrain.data.addData(inputs, [parseFloat(freq)]); + } + } } function getInputs() { @@ -85,15 +82,14 @@ function getInputs() { } } -function addExample() { - let freq = select('#frequency_slider').value(); - let inputs = getInputs(); - if (inputs) { - faceBrain.data.addData(inputs, [parseFloat(freq)]); - } +function collectData() { + osc.start(); + osc.amp(5); + collecting = true; } function trainModel() { + collecting = false; osc.amp(0); faceBrain.data.normalize(); const trainingOptions = {