ml5js · shiffman · Sep 29, 2019 · Oct 1, 2019 · Oct 1, 2019 · Oct 1, 2019
diff --git a/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/index.html b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/index.html
@@ -0,0 +1,27 @@
+<html>
+
+<head>
+  <meta charset="UTF-8">
+  <title>Neural Network Sound Player</title>
+
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/p5.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.dom.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.sound.min.js"></script>
+  <script src="http://localhost:8080/ml5.js" type="text/javascript"></script>
+</head>
+
+<body>
+  <h1>Pixel Prediction</h1>
+  <script src="sketch.js"></script>
+
+  <p>
+    Training frequency: <input id="frequency" value="440" />
+    <button id="addExample">add example</button>
+  </p>
+  <p>
+    <button id="train">train model</button>
+    Frequency prediction: <span id="prediction"></span>
+  </p>
+</body>
+
+</html>
diff --git a/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_lowres_pixels/sketch.js
@@ -0,0 +1,115 @@
+
+
+let pixelBrain;
+let video;
+let ready = false;
+let w;
+let playing = false;
+let frequency;
+let osc;
+
+function setup() {
+  createCanvas(200, 200);
+  video = createCapture(VIDEO, videoReady);
+  let res = 10;
+  video.size(res, res);
+  video.hide();
+  let totalPixels = res * res * 3;
+  const options = {
+    inputs: totalPixels,
+    outputs: 1,
+    hiddenUnits: floor(totalPixels / 2),
+    // activationHidden: 'relu',
+    learningRate: 0.01,
+    debug: true,
+  }
+  pixelBrain = ml5.neuralNetwork(options);
+  select('#addExample').mousePressed(addExample);
+  select('#train').mousePressed(trainModel);
+  w = width / res;
+  osc = new p5.Oscillator();
+  osc.setType('sine');
+  osc.amp(0.5);
+  osc.freq(440);
+}
+
+function videoReady() {
+  ready = true;
+}
+
+function draw() {
+  background(0);
+  if (ready) {
+    video.loadPixels();
+    for (let x = 0; x < video.width; x++) {
+      for (let y = 0; y < video.height; y++) {
+        let index = (x + y * video.width) * 4;
+        let r = video.pixels[index + 0];
+        let g = video.pixels[index + 1];
+        let b = video.pixels[index + 2];
+        noStroke();
+        fill(r, g, b);
+        rect(x * w, y * w, w, w);
+      }
+    }
+  }
+
+}
+
+function getInputs() {
+  video.loadPixels();
+  let inputs = [];
+  for (let i = 0; i < video.width * video.height; i++) {
+    let index = i * 4;
+    inputs.push(video.pixels[index + 0]);
+    inputs.push(video.pixels[index + 1]);
+    inputs.push(video.pixels[index + 2]);
+  }
+  return inputs;
+}
+
+let firstTime = true;
+function addExample() {
+  if (firstTime) {
+    osc.start();
+    firstTime = false;
+  }
+
+  let freq = select('#frequency').value();
+  osc.freq(parseFloat(freq));
+  video.loadPixels();
+  let inputs = getInputs();
+  pixelBrain.data.addData(inputs, [parseFloat(freq)]);
+}
+
+function trainModel() {
+  osc.amp(0);
+  pixelBrain.data.normalize();
+  const trainingOptions = {
+    epochs: 50
+  }
+  pixelBrain.train(trainingOptions, finishedTraining);
+}
+
+function finishedTraining() {
+  console.log('done');
+  osc.amp(0.5);
+  predict();
+}
+
+function predict() {
+  let inputs = getInputs();
+  pixelBrain.predict(inputs, gotFrequency);
+}
+
+function gotFrequency(error, results) {
+  if (error) {
+    console.error(error);
+  } else {
+    frequency = parseFloat(results.outputs.value);
+    select('#prediction').html(frequency.toFixed(2));
+    osc.freq(parseFloat(frequency));
+    predict();
+  }
+}
+
diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html b/p5js/NeuralNetwork/NeuralNetwork_musical_face/index.html
@@ -0,0 +1,28 @@
+<html>
+
+<head>
+  <meta charset="UTF-8">
+  <title>Neural Network Sound Player</title>
+
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/p5.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.dom.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.sound.min.js"></script>
+  <script src="http://localhost:8080/ml5.js" type="text/javascript"></script>
+</head>
+
+<body>
+  <h1>Face Prediction</h1>
+  <script src="sketch.js"></script>
+
+  <p>
+    Training frequency: <span id="training_freq">440</span>
+    <input type="range" id="frequency_slider" min="0" max="800" value="440" />
+    <button id="collectData">collect training data</button>
+  </p>
+  <p>
+    <button id="train">train model</button>
+    Frequency prediction: <span id="prediction"></span>
+  </p>
+</body>
+
+</html>
diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_musical_face/sketch.js
@@ -0,0 +1,121 @@
+let faceapi;
+let video;
+let detections;
+
+let faceBrain;
+let osc;
+let trained = false;
+let collecting = false;
+
+
+function setup() {
+  createCanvas(360, 270);
+  video = createCapture(VIDEO);
+  video.size(width, height);
+  // video.hide();
+
+  const faceOptions = {
+    withLandmarks: true,
+    withExpressions: false,
+    withDescriptors: false,
+  }
+  faceapi = ml5.faceApi(video, faceOptions, modelReady);
+
+  const options = {
+    inputs: 68 * 2,
+    outputs: 1,
+    hiddenUnits: 68,
+    learningRate: 0.01,
+    debug: true,
+  }
+  faceBrain = ml5.neuralNetwork(options);
+  select('#collectData').mousePressed(collectData);
+  select('#train').mousePressed(trainModel);
+
+  osc = new p5.Oscillator();
+  osc.setType('sine');
+}
+
+function modelReady() {
+  faceapi.detect(gotResults);
+}
+
+function gotResults(err, result) {
+  if (err) {
+    console.log(err);
+    return;
+  }
+  detections = result;
+  faceapi.detect(gotResults);
+}
+
+function draw() {
+  background(0);
+  if (detections && detections.length > 0) {
+    let points = detections[0].landmarks.positions;
+    for (let i = 0; i < points.length; i++) {
+      stroke(161, 95, 251);
+      strokeWeight(4);
+      point(points[i]._x, points[i]._y);
+    }
+  }
+  if (collecting) {
+    let freq = select('#frequency_slider').value();
+    select('#training_freq').html(freq);
+    osc.freq(freq);
+    let inputs = getInputs();
+    if (inputs) {
+      faceBrain.data.addData(inputs, [parseFloat(freq)]);
+    }
+  }
+}
+
+function getInputs() {
+  if (detections && detections.length > 0) {
+    let points = detections[0].landmarks.positions;
+    let inputs = [];
+    for (let i = 0; i < points.length; i++) {
+      inputs.push(points[i]._x);
+      inputs.push(points[i]._y);
+    }
+    return inputs;
+  }
+}
+
+function collectData() {
+  osc.start();
+  osc.amp(5);
+  collecting = true;
+}
+
+function trainModel() {
+  collecting = false;
+  osc.amp(0);
+  faceBrain.data.normalize();
+  const trainingOptions = {
+    epochs: 50
+  }
+  faceBrain.train(trainingOptions, finishedTraining);
+}
+
+function finishedTraining() {
+  console.log('done');
+  osc.amp(0.5);
+  predict();
+}
+
+function predict() {
+  let inputs = getInputs();
+  faceBrain.predict(inputs, gotFrequency);
+}
+
+function gotFrequency(error, results) {
+  if (error) {
+    console.error(error);
+  } else {
+    frequency = parseFloat(results.outputs.value);
+    osc.freq(frequency);
+    select('#prediction').html(frequency.toFixed(2));
+    predict();
+  }
+}
diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_mouse/index.html b/p5js/NeuralNetwork/NeuralNetwork_musical_mouse/index.html
@@ -0,0 +1,31 @@
+<html>
+
+<head>
+  <meta charset="UTF-8">
+  <title>Neural Network Sound Player</title>
+
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/p5.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.dom.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.sound.min.js"></script>
+  <script src="http://localhost:8080/ml5.js" type="text/javascript"></script>
+</head>
+
+<body>
+  <h1>Neural Network Sound Player</h1>
+  <script src="sketch.js"></script>
+
+  <p>
+    Training frequency: <input id="frequency" value="440" />
+    <button id="train">train</button>
+  </p>
+  <p>
+    Frequency prediction: <span id="prediction"></span>
+  </p>
+
+  <p>
+    Click in canvas to add training data.
+  </p>
+
+</body>
+
+</html>
diff --git a/p5js/NeuralNetwork/NeuralNetwork_musical_mouse/sketch.js b/p5js/NeuralNetwork/NeuralNetwork_musical_mouse/sketch.js
@@ -0,0 +1,62 @@
+let notePlayer;
+let playing = false;
+let frequency;
+let osc;
+
+function setup() {
+  createCanvas(400, 400).mousePressed(addData);;
+  const options = {
+    inputs: 2,  // what about allowing ['x', 'y']?
+    outputs: 1, // what about allowing ['x', 'y']?
+    debug: true,
+  }
+  background(0);
+  notePlayer = ml5.neuralNetwork(options);
+  select('#train').mousePressed(trainModel);
+}
+
+function addData() {
+
+  let freq = select('#frequency').value();
+  stroke(255);
+  noFill();
+  ellipse(mouseX, mouseY, 32);
+  fill(255);
+  textSize(16);
+  console.log(freq);
+  textAlign(CENTER, CENTER);
+  text(freq, mouseX, mouseY);
+  notePlayer.data.addData([mouseX, mouseY], [parseFloat(freq)]);
+}
+
+function trainModel() {
+  notePlayer.data.normalize();
+  const trainingOptions = {
+    batchSize: 24,
+    epochs: 20
+  }
+  notePlayer.train(trainingOptions, finishedTraining);
+}
+
+function finishedTraining() {
+  console.log('done');
+  osc = new p5.Oscillator();
+  osc.setType('sine');
+  osc.amp(0.5);
+  osc.freq(440);
+  osc.start();
+  notePlayer.predict([mouseX, mouseY], gotFrequency);
+}
+
+function gotFrequency(error, results) {
+  if (error) {
+    console.error(error);
+  } else {
+    frequency = parseFloat(results.outputs.value);
+    console.log(results);
+    select('#prediction').html(frequency.toFixed(2));
+    osc.freq(parseFloat(frequency));
+    notePlayer.predict([mouseX, mouseY], gotFrequency);
+  }
+}
+