Detecting eye color using PoseNet

p5.js Libraries
1

Building on Dan Shiffman’s PoseNet example, I built a model to get eye color using the left and right eye points. I applied a 6 pixel offset from each point’s center to avoid the pupil and, ideally, land on the iris. It seems to work, but I would be grateful for feedback and suggestions as to how to improve the performance.

let video;
let poseNet;
let pose;
let eyeLcolor = [];
let eyeRcolor = [];

function setup() {
  createCanvas(640, 480);
  video = createCapture(VIDEO);
  video.hide();
  poseNet = ml5.poseNet(video, modelLoaded);
  poseNet.on('pose', gotPoses);
}

function gotPoses(poses) {
  // console.log(poses);
  if (poses.length > 0) {
    pose = poses[0].pose;
  }
}

function draw() {
  image(video, 0, 0);
  if (pose) {
    let eyeR = pose.rightEye;
    let eyeL = pose.leftEye;

    noFill();
    stroke(0, 255, 0);
    ellipse(eyeL.x, eyeL.y, 20, 20);
    ellipse(eyeR.x, eyeR.y, 20, 20);

    let eyeLcolor = get(eyeL.x - 6, eyeL.y);
    let eyeRcolor = get(eyeR.x + 6, eyeR.y);
    cL = color(eyeLcolor);
    cR = color(eyeRcolor);

    noStroke();
    fill(cL)
    ellipse(50, 50, 80, 80);
    fill(cR);
    ellipse(150, 50, 80, 80);

    textSize(16);
    fill(200);
    textAlign(CENTER);
    text("LEFT", 50, 120);
    text("RIGHT", 150, 120);

    // print(eyeLcolor);
    // print(eyeRcolor);
  }
}

function modelLoaded() {
  console.log(eyeLcolor);
  console.log(eyeRcolor);
}
2

Very neat! I’ve never used PoseNet, but always be weary of using constants to capture user input. For instance, what if the radius of the iris’s are larger than 6px?

3

Thanks a lot for your feedback Tony. Your point about constants is appreciated; the diameter of the iris ranges from 10-13 mm which is why I chose 6. Perhaps I could widen the detection area a few mm using and then identify the average color between the black pupil and white sclera. Would also like to label the color with a name (rather than just RGB values) in a future version.

https://editor.p5js.org/gstilwell/full/GyKQ0-Kuh