*Almost* combining brightness example and posenet

p5.js Libraries
1

Hi everyone. I am trying to modify the brightness example:

https://p5js.org/examples/color-brightness.html

to combine it with ml5 posenet. Instead of using the mouse position to move around the image I want to use the position of the viewer’s face.

I’ve managed to isolate the nose point by following some of the coding train examples. After doing so I slowly added the elements into the brightness example. Things were playing nicely together until the very end. When I zipped everything up I got the following error in the console:

TypeError: Argument 1 is not valid for any of the 1-argument overloads of URL.createObjectURL. p5.dom.min.js:1:7286

This wasn’t super helpful to me because I am not familiar with what is going on at line 7,286 of the p5 dom library.

I have pasted the code below and would love a pointer to where I am going wrong. Even though it makes things a bit longer I have included the commented out parts of the draw() function where I put each of the bits (the brightness and ml5js) in sequence just in case it makes it easier to see what I’m trying to do. The ‘real’ part of the code is the part under the combined version comment. It is, you know, where I tried to combine the bits.

Thank you in advance for any help!

let img;
let video;
let poseNet;
let pose;
let skeleton;

function preload() {
  img = loadImage('assets/flower0.jpg');
}

function setup() {
    //brightness stuff
  createCanvas(720, 720);
  pixelDensity(1);
  img.loadPixels();
  loadPixels();

  //ml5 stuff
  video = createCapture(VIDEO);
  video.hide();
  poseNet = ml5.poseNet(video, modelLoaded);
  poseNet.on('pose', gotPoses);
}

function gotPoses(poses) {
  console.log(poses);
  if (poses.length > 0) {
    pose = poses[0].pose;
  }
}

function modelLoaded() {
  console.log("model loaded");
}

function draw() {
//combined version

    if (pose) {
        let noseX = pose.nose.x;
        let noseY = pose.nose.y;
        for (let x = 0; x < img.width; x++) {
          for (let y = 0; y < img.height; y++) {
            // Calculate the 1D location from a 2D grid
            let loc = (x + y * img.width) * 4;
            // Get the R,G,B values from image
            let r, g, b;
            r = img.pixels[loc];
            // Calculate an amount to change brightness based on proximity to the mouse
            let maxdist = 50;
            let d = dist(x, y, noseX, noseY);
            let adjustbrightness = (255 * (maxdist - d)) / maxdist;
            r += adjustbrightness;
            // Constrain RGB to make sure they are within 0-255 color range
            r = constrain(r, 0, 255);
            // Make a new color and set pixel in the window
            //color c = color(r, g, b);
            let pixloc = (y * width + x) * 4;
            pixels[pixloc] = r;
            pixels[pixloc + 1] = r;
            pixels[pixloc + 2] = r;
            pixels[pixloc + 3] = 255;
          }
        }
    }



  //   //brightness part
  // for (let x = 0; x < img.width; x++) {
  //   for (let y = 0; y < img.height; y++) {
  //     // Calculate the 1D location from a 2D grid
  //     let loc = (x + y * img.width) * 4;
  //     // Get the R,G,B values from image
  //     let r, g, b;
  //     r = img.pixels[loc];
  //     // Calculate an amount to change brightness based on proximity to the mouse
  //     let maxdist = 50;
  //     let d = dist(x, y, mouseX, mouseY);
  //     let adjustbrightness = (255 * (maxdist - d)) / maxdist;
  //     r += adjustbrightness;
  //     // Constrain RGB to make sure they are within 0-255 color range
  //     r = constrain(r, 0, 255);
  //     // Make a new color and set pixel in the window
  //     //color c = color(r, g, b);
  //     let pixloc = (y * width + x) * 4;
  //     pixels[pixloc] = r;
  //     pixels[pixloc + 1] = r;
  //     pixels[pixloc + 2] = r;
  //     pixels[pixloc + 3] = 255;
  //   }
  // }
  // updatePixels();
  //
  // //ml5 part
  // if (pose) {
  //
  //   fill(255,0,0);
  //   let noseX = pose.nose.x;
  //   let noseY = pose.nose.y;
  //   ellipse(noseX, noseY, 50);
  //   }
}
1 Like
2

Hey, I have a similar working example.

See it in action here: https://editor.p5js.org/AndreasRef/sketches/LMneBrcD

Does that do the trick?

let video;
let poseNet;
let poses = [];

let nose; 

function setup() {
  createCanvas(640, 480);
  video = createCapture(VIDEO);
  video.size(width, height);
  pixelDensity(1);
  nose = createVector(0,0);

  // Create a new poseNet method with a single detection
  poseNet = ml5.poseNet(video, modelReady);
  // This sets up an event that fills the global variable "poses"
  // with an array every time new poses are detected
  poseNet.on('pose', function(results) {
    poses = results;
  });
  // Hide the video element, and just show the canvas
  video.hide();
  
}

function modelReady() {
  select('#status').html('Model Loaded');
}

function mousePressed(){
  console.log(JSON.stringify(poses[0].pose.keypoints[0].part))
}

function draw() {
  
  image(video, 0, 0, width, height);

  if (poses.length > 0) {
    let pose = poses[0].pose;

    nose = pose['nose'];
  } 
  
  loadPixels();
    for (let x = 0; x < width; x++) {
    for (let y = 0; y < height; y++) {
      // Calculate the 1D location from a 2D grid
      let loc = (x + y * width) * 4;
      // Get the R,G,B values from image
      let r, g, b;
      r = pixels[loc];
      // Calculate an amount to change brightness based on proximity to the mouse
      let maxdist = 50;
      let d = dist(x, y, nose.x, nose.y);
      let adjustbrightness = (255 * (maxdist - d)) / maxdist;
      r += adjustbrightness;
      // Constrain RGB to make sure they are within 0-255 color range
      r = constrain(r, 0, 255);
      // Make a new color and set pixel in the window
      //color c = color(r, g, b);
      let pixloc = (y * width + x) * 4;
      pixels[pixloc] = r;
      pixels[pixloc + 1] = r;
      pixels[pixloc + 2] = r;
      pixels[pixloc + 3] = 255;
    }
  }
  updatePixels();
}
2 Likes
3

First off, this is awesome and makes me think about doing all sorts of other interesting things with this. Thank you for sharing it.

Second, very close to yes. How would I modify your code to load a static image instead of the live video? I still want the spotlight to track the nose, I just want it to be revealing an image instead of the person.

thank you!

1 Like
4

No problem :slight_smile:

Yeah, then suppose I would load a static image and display it in draw function instead of displaying the video (or just after image(video, 0, 0, width, height); which would effectively hide the webcam stream behind your video).

1 Like
5

That did it! I had been trying to do that with my code but clearly missing some key element. It may have been that I was not loading the image in the setup() function as well. Whatever it was, thank you so much for the help.

1 Like