Help in understanding the tensorflow error for mediapipe hands estimation

p5.js Libraries
1

Hi, I have been trying to follow the mediapipe documentation and associated tutorial to check if the handpose estimation is working. I seem to get into the following error . I am unable to understand what is the reason for the error ? I am trying to run the script in p5jsweb editor Is it that the webgl backend is not setup in time ?

The link to the API is as follows : Media Pipe Hand Js API.
Error log

RuntimeError: abort(undefined) at Error
    at jsStackTrace (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:69361)

    at stackTrace (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:69537)

    at abort (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:43094)

    at _abort (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:180746)

    at https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[7269]:0x4ff31e

    at https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[7267]:0x4ff1c3

    at https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[7271]:0x4ff337

    at https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[222]:0x19c2c

    at https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[192]:0x14cd2

    at SolutionWasm$send [as send] (eval at new_ (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:161520), <anonymous>:9:1)

    at jsStackTrace (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:69361)

    at stackTrace (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:69537)

    at abort (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:43094)

    at _abort (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.js:9:180746)

    at (anonymous function) (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[7269]:0x4ff31e)

    at (anonymous function) (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[7267]:0x4ff1c3)

    at (anonymous function) (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[7271]:0x4ff337)

    at (anonymous function) (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[222]:0x19c2c)

    at (anonymous function) (https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands_solution_simd_wasm_bin.wasm:wasm-function[192]:0x14cd2)

    at SolutionWasm$send [as send]

I have four files index.html, sketch.js, detection.js. style.css

index.html

<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js" crossorigin="anonymous"></script>

  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.2.0/p5.js"></script>

  <script defer language="javascript" type="text/javascript" src="detection.js"></script>
  <script defer language="javascript" type="text/javascript" src="sketch.js"></script>
  <link rel="stylesheet" type="text/css" href="style.css">
</head>

<body>
  <video id="video" width="640" height="480"></video>
</body>
</html>
 
   <body>
  <div class="container">
    <video class="input_video"></video>
    <canvas class="output_canvas" width="640px" height="480px"></canvas>
  </div>
</body> 
   
</html>

sketch.js



let canvas;

let sketch = function(p){
  p.setup = function(){
    canvas = p.createCanvas(640, 480);
    canvas.id("canvas");

    p.colorMode(p.HSB);
  }

  p.draw = function(){
    p.clear();
    if(detections != undefined){
      if(detections.multiHandLandmarks != undefined){
          //p.drawHands();
          // p.drawParts();

          p.drawLines([0, 5, 9, 13, 17, 0]);//palm
          p.drawLines([0, 1, 2, 3 ,4]);//thumb
          p.drawLines([5, 6, 7, 8]);//index finger
          p.drawLines([9, 10, 11, 12]);//middle finger
          p.drawLines([13, 14, 15, 16]);//ring finger
          p.drawLines([17, 18, 19, 20]);//pinky

          p.drawLandmarks([0, 1], 0);//palm base
          p.drawLandmarks([1, 5], 60);//thumb
          p.drawLandmarks([5, 9], 120);//index finger
          p.drawLandmarks([9, 13], 180);//middle finger
          p.drawLandmarks([13, 17], 240);//ring finger
          p.drawLandmarks([17, 21], 300);//pinky
      }
    }
  }

  p.drawHands = function(){
    for(let i=0; i<detections.multiHandLandmarks.length; i++){
      for(let j=0; j<detections.multiHandLandmarks[i].length; j++){
        let x = detections.multiHandLandmarks[i][j].x * p.width;
        let y = detections.multiHandLandmarks[i][j].y * p.height;
        let z = detections.multiHandLandmarks[i][j].z;
        // p.strokeWeight(0);
        // p.textFont('Helvetica Neue');
        // p.text(j, x, y);
        p.stroke(255);
        p.strokeWeight(10);
        p.point(x, y);
      }
    }
  }

  p.drawLandmarks = function(indexArray, hue){
    p.noFill();
    p.strokeWeight(8);
    for(let i=0; i<detections.multiHandLandmarks.length; i++){
      for(let j=indexArray[0]; j<indexArray[1]; j++){
        let x = detections.multiHandLandmarks[i][j].x * p.width;
        let y = detections.multiHandLandmarks[i][j].y * p.height;
        // let z = detections.multiHandLandmarks[i][j].z;
        p.stroke(hue, 40, 255);
        p.point(x, y);
      }
    }
  }

  p.drawLines = function(index){
    p.stroke(0, 0, 255);
    p.strokeWeight(3);
    for(let i=0; i<detections.multiHandLandmarks.length; i++){
      for(let j=0; j<index.length-1; j++){
        let x = detections.multiHandLandmarks[i][index[j]].x * p.width;
        let y = detections.multiHandLandmarks[i][index[j]].y * p.height;
        // let z = detections.multiHandLandmarks[i][index[j]].z;

        let _x = detections.multiHandLandmarks[i][index[j+1]].x * p.width;
        let _y = detections.multiHandLandmarks[i][index[j+1]].y * p.height;
        // let _z = detections.multiHandLandmarks[i][index[j+1]].z;
        p.line(x, y, _x, _y);
      }
    }
  }
}

let myp5 = new p5(sketch);

detection.js


let detections = {};

const videoElement = document.getElementById('video');

const hands = new Hands({locateFile: (file) => {
  return `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`;
}});

hands.setOptions({
  maxNumHands: 4,
  minDetectionConfidence: 0.8,
  minTrackingConfidence: 0.5
});

hands.onResults(gotHands);

function gotHands(results) {
  detections = results;
  console.log(detections);
}

const camera = new Camera(videoElement, {
  onFrame: async () => {
    await hands.send({image: videoElement});
  },
  width: 640,
  height: 480
});
camera.start(); ```
2

Try putting let detections = {}; at the top of your sketch.js. That stops the error messages for me but it still doesn’t run.

3

Initially I didn’t see the canvas when run. You sketch.js source code uses p.setup = function() and p.draw = function() which is an instance mode. There are quite a few syntax errors according to the documentation: https://p5js.org/reference/#/p5/p5. After fixing these errors I can now see the canvas, but unfortunately not the camera image.

4

Thanks for the tips. I will have a look into it. I was actually following the following tutorial here .

P5js mediapipe framework and trying to run the code from here to get an overall understanding of how things work . Code .Perhaps should try to understand the flow better.

5

The video is pretty slick and for me very complicated. I don’t know what your skill level is, but you might want to start with the code at the very beginning of the video. The GitHub repository is probably the sum of all of his many modifications. My system choked on the async() call in detection.js and I notice he had no problem with this on his system. I was never able to open the webcam. I wish he had run the code in Processing IDE or one of the popular web editors in lieu of Atom. It’s cool technology; not sure where it fits in the grand scheme of things, but definitely cool nonetheless. Thanks for posting it.

6

Hello yes, I was also struggling in the async part and I think that is the key. Yes too bad he had not tried in on a online IDE. There could be some issues on running it on web editor .

7

@svan , do you have any suggestions on these as in how to tackle the problem

8

I think the ‘async’ error is a major hurdle for getting this project to run. There are easier ways to activate a webcam, but then the remainder of the code would have to somehow be connected to the alternate technique. The error message states that our browser needs esversion 8, also known as ES8, or ES2017. I think current browsers are back at about ES6 or ES7. These are all versions of ECMAScript, which originated in the 1990’s in an early browser called Netscape. It is possible to download a newer version, but I have no idea how to get it into my current browser (or if that is even possible). The newer version has to be usable somewhere, but how? This is perhaps a good question for more knowledgeable members of our community. The only other option that I see is to use the easy way to connect to a webcam and give up on ‘async’. However, I don’t know if the other code will work correctly without it.

9

The following source code will run in the p5.js web editor using a Chrome browser (will not run in Safari on Mac):

index.html

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="utf-8">
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js" crossorigin="anonymous"></script>

    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/p5.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/addons/p5.sound.min.js"></script>
    <link rel="stylesheet" type="text/css" href="style.css">
    <meta charset="utf-8" />

  </head>
  <body>
       
     <div class="container">
    <video class="input_video"></video>
    <canvas class="output_canvas" width="640px" height="480px"></canvas>
  </div>
    <script src="sketch.js"></script>
  </body>
</html>

sketch.js

const videoElement = document.getElementsByClassName('input_video')[0];
const canvasElement = document.getElementsByClassName('output_canvas')[0];
const canvasCtx = canvasElement.getContext('2d');

function onResults(results) {
  canvasCtx.save();
  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
  canvasCtx.drawImage(
      results.image, 0, 0, canvasElement.width, canvasElement.height);
  if (results.multiHandLandmarks) {
    for (const landmarks of results.multiHandLandmarks) {
      drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS,
                     {color: '#00FF00', lineWidth: 5});
      drawLandmarks(canvasCtx, landmarks, {color: '#FF0000', lineWidth: 2});
    }
  }
  canvasCtx.restore();
}

const hands = new Hands({locateFile: (file) => {
  return `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`;
}});
hands.setOptions({
  maxNumHands: 2,
  modelComplexity: 1,
  minDetectionConfidence: 0.5,
  minTrackingConfidence: 0.5
});
hands.onResults(onResults);

const camera = new Camera(videoElement, {
  onFrame: async () => {
    await hands.send({
      image: videoElement
    });
  },
  width: 640,
  height: 480
});
camera.start();

output:

hands
hands778×358 252 KB

10

Hi wow nice that it works for your setup. Apparently it does not work on my chrome setup. it throws no error (surprisingly) so i am not sure what could be the problem is

11

It has to be run in p5.js web editor. What operating system are you using?

12

hi it is running on a windows machine , on chrome web browser. So i am not sure what could be the error here. i can run the code pen example but not the p5js script

I can try to run it on another machine just to be sure its not my machine.

13

I used a Mac, but that shouldn’t make any difference. Do you have your chrome browser set to open p5.js web editor? When it opens copy/paste the code that I posted into a new sketch.

14

Hi i exactly did that , copy pasted the code into my webeditor in p5js. Can you share the sketch directly so that i can check if it works ? I am using chrome as you had suggested.

15

Try this link:
https://editor.p5js.org/svanvrst/full/hRrXe2FRm

or this one for edit:
https://editor.p5js.org/svanvrst/sketches/hRrXe2FRm

16

Thanks it seems to work , i was under the impression that the output was overlaid on the original one and perhaps while trying that i missed the rendered video. I guess it could be overlaid on top of the original video . I will have to look into that.