How can I make my code work for multiple figure detection (poseNet)?

Hoi!

I’ve just started learning how to code and in my school, there’s not much guidance about the subject. Currently, I’m doing a project with p5.js and poseNet. But I’m stuck trying to modify each keypoint independently, it’s only working with one figure at a time. How can I command the keypoints of multiple subjects at the same time? Many thanks!

let video;
let poseNet;
let poses = [];
//let maxPoseDetections = 5;


let noseX = 0;
let noseY = 0;
let leftShoulderX = 0;
let leftShoulderY = 0;
let rightShoulderX = 0;
let rightShoulderY = 0;
let leftElbowX = 0;
let leftElbowY = 0;
let rightElbowX = 0;
let rightElbowY = 0;
let leftWristX = 0;
let leftWristY = 0;
let rightWristX = 0;
let rightWristY = 0;

 
function setup() {
  createCanvas(640, 480);
  video = createCapture(VIDEO);
  video.size(width, height);
  
  poseNet = ml5.poseNet(video, modelReady);
  poseNet.on('pose', gotPoses);
  poseNet.on('pose', function (results) {
  poses = results;
  });
  video.hide();
}

function modelReady() {
  console.log('model ready');
}

function draw() {
  image(video, 0, 0, width, height);
  drawKeypoints();
  //drawSkeleton();
}

function gotPoses(poses) {
  //console.log(poses);
  if (poses.length > 0) {
    let nX = poses[0].pose.keypoints[0].position.x;
    let nY = poses[0].pose.keypoints[0].position.y;
    let lsX = poses[0].pose.keypoints[5].position.x;
    let lsY = poses[0].pose.keypoints[5].position.y;
    let rsX = poses[0].pose.keypoints[6].position.x;
    let rsY = poses[0].pose.keypoints[6].position.y;
    let leX = poses[0].pose.keypoints[7].position.x;
    let leY = poses[0].pose.keypoints[7].position.y;
    let reX = poses[0].pose.keypoints[8].position.x;
    let reY = poses[0].pose.keypoints[8].position.y;
    let lwX = poses[0].pose.keypoints[9].position.x;
    let lwY = poses[0].pose.keypoints[9].position.y;
    let rwX = poses[0].pose.keypoints[10].position.x;
    let rwY = poses[0].pose.keypoints[10].position.y;

    noseX = lerp(noseX, nX, 0.4);
    noseY = lerp(noseY, nY, 0.4);
    leftShoulderX = lerp(leftShoulderX, lsX, 0.3);
    leftShoulderY = lerp(leftShoulderY, lsY, 0.3);
    rightShoulderX = lerp(rightShoulderX, rsX, 0.3);
    rightShoulderY = lerp(rightShoulderY, rsY, 0.3);
    leftElbowX = lerp(leftElbowX, leX, 0.5);
    leftElbowY = lerp(leftElbowY, leY, 0.5);
    rightElbowX = lerp(rightElbowX, reX, 0.5);
    rightElbowY = lerp(rightElbowY, reY, 0.5);
    leftWristX = lerp(leftWristX, lwX, 0.5);
    leftWristY = lerp(leftWristY, lwY, 0.5);
    rightWristX = lerp(rightWristX, rwX, 0.5);
    rightWristY = lerp(rightWristY, rwY, 0.5);
  }
}

function drawKeypoints() {
  for (let i = 0; i < poses.length; i++) {
  let pose = poses[i].pose;
    for (let j = 0; j < pose.keypoints.length; j++) {
  let keypoint = pose.keypoints[j];
  if (noseX, noseY, leftShoulderX, leftShoulderY, rightShoulderX, rightShoulderY, leftElbowX, rightElbowX, leftWristX, leftWristY  > 0.4) {
  //image(video, 0, 0, width, height)
  //filter(INVERT)
  //let d = dist(noseX, noseY, leftShoulderY, leftShoulderX);
  
  noStroke();
  // blue
  fill(0, 0, 255);
  ellipse(noseX, noseY, 10);
  // red
  fill(255,0,0);
  ellipse(leftShoulderX, leftShoulderY, 10);
  fill(255,0,0);
  ellipse(rightShoulderX, rightShoulderY, 10);
  // red
  fill(255,0,0);
  ellipse(leftElbowX, leftElbowY, 10);
  fill(255,0,0);
  ellipse(rightElbowX, rightElbowY, 10);
  // green
  fill(0,255,0);
  ellipse(leftWristX, leftWristY, 10);
  fill(0,255,0);
  ellipse(rightWristX, rightWristY, 10);
      }
    }
  }
}
1 Like

If you post your question to multiple sites, please link between those posts so we know what help you’ve already received.

This question has also been posted here:

1 Like

Ok, thanks for letting me know :slight_smile:

1 Like

What do you mean by this? Do you mean that you can modify poses[0], but cannot modify poses[1]?

The keypoints only track one person, when someone else appears on the video feed the code won’t generate a second set of keypoints. So it ends up shuffling between them.

I hope that was clear enough Jeremy :slight_smile:

Right – my question was, is posenet returning multiple poses at all? Does a second pose, poses[1], exist? If it does, then you can change your code to work with it – but that is the first question.

Next – when you say “work” for multiple figures – what do you want to do with each of them?

Not entirely sure how can I check if poseNet is returning multiple poses. I know there’s a version of the algorithm that works with single detection and one with multiple, but that’s as far as my knowledge goes, and the code :confused:

The “let nX = poses[0].pose.keypoints[0].position.x;” was taken from this tutorial.

I’m trying to assign a different color to each key point of every figure detected, for a design project. Later I want to make an Arduino send those points to a led panel. Really thanks for your time!!

having the same problem hit me up when u find smth