setInterval with Posenet

p5.js Libraries
1

I have a quick question about using Posenet! I’m coding a website like you can see here: https://distracted-euclid-963531.netlify.app/ that uses body movement to process sound. Clicking the button starts a looped sound file that you can pan left and right using your nose and can reverberate by moving your right wrist horizontally. Additionally, I’m trying to work through using the left wrist’s vertical position to delay a sound file which you can hear if you hold your left wrist up in the frame and click on the left arrow on your keyboard. I’m having a hard time trying to set the delay to only occur every few seconds even with using the setInterval() code. I logged the delay time to the console but it looks like it’s constantly looping through the delay code rather than doing it every I’ve attached my js code below if anyone can shed some knowledge.

let video;
let poseNet;
let poses = [];
let skeletons = [];
let nose, rightWrist, leftWrist, rightKnee, leftKnee;
let nScore, rwScore, lwScore, rkScore, lkScore;
let song, song2, song3;
let button, button2;
let fft, noise, fil;
let delay, rev;

function preload() {
 soundFormats('mp3', 'ogg', 'wav');
 song = loadSound('dhol_main_loop.mp3');
 song2 = loadSound('algoze_shorter.wav');
}

function modelLoaded() {
 console.log("Model Loaded");
}

function setup() {
 cnv = createCanvas(640, 480);
 video = createCapture(VIDEO);
 video.size(width, height);

 button = document.getElementById("start1");
 button.addEventListener("click", handleButton1Click);

 /*
 button2 = document.getElementById("start2");
 button2.addEventListener("click", handleButton2Click);
 */

 poseNet = ml5.poseNet(video, modelLoaded);
 poseNet.on('pose', getPoses);
 video.hide();

 delay = new p5.Delay();
 song2.connect();
 delay.process(song2, 0.12, 0.5, 2300);
 delay.setType('pingPong');
 delay.delayTime(0.0);
 delay.amp(0.0);
 //console.log(delay);

 rev = new p5.Reverb();
 song.connect();
 rev.process(song, 3, 2);
}

 function draw() {    
   //mirrors image so you're viewing 
   translate(video.width, 0);
   scale(-1, 1);
   image(video, 0, 0, video.width, video.height);
 
   //draws dots on nose, wrists, and knees 
   drawPoints();

   //delays and filters audio with wrist movement
   if(leftWrist) {
     setInterval(delayAudio, 3000);
   } else {
     return;
   }

   //pans audio by following nose movement 
   panning();

   //reverb and then panning
   if(rightWrist) {
     reverbAudio();
   }

   //delay into reverb into panning

   //cut out all other audio while triggering one sound
   //might be smarter to use quadrants and body part positioning 
   //
 }

function getPoses(poses) {
 if(poses.length > 0) {
   let pose = poses[0].pose;

   nose = pose.keypoints[0].position;
   nScore = pose.keypoints[0].score;

   leftWrist = pose.keypoints[9].position;
   lwScore = pose.keypoints[9].score;

   rightWrist = pose.keypoints[10].position;
   rwScore = pose.keypoints[10].score;

   leftKnee = pose.keypoints[13].position;
   lkScore = pose.keypoints[13].score;

   rightKnee = pose.keypoints[14].position;
   rkScore = pose.keypoints[14].score;
 }
}

function drawPoints() {
 if(nose && nScore > 0.6) {
   stroke(255, 0, 0);
   strokeWeight(15);
   point(nose.x, nose.y);
 }

 if(rightWrist && rwScore > 0.3) {
   stroke(0, 0, 255);
   strokeWeight(15);
   point(rightWrist.x, rightWrist.y); 
 }

 if(leftWrist && lwScore > 0.05) {
   stroke(0, 255, 255);
   strokeWeight(15);
   point(leftWrist.x, leftWrist.y);
 }

 if((rightKnee && leftKnee) && (rkScore && lkScore > 0.6)) {
   stroke(0, 255, 0);
   strokeWeight(15);
   point(rightKnee.x, rightKnee.y);
   stroke(0, 255, 0);
   strokeWeight(15);
   point(leftKnee.x, leftKnee.y);
 }
}

function handleButton1Click() {
 if(song.isLooping()) {
   button.innerText = "Start Dhol";
   song.stop();
 } else {
   button.innerText = "Stop Dhol";
   song.loop();
 }
}

/*
function handleButton2Click() {
 if(song2.isLooping()) {
   button2.innerText = "Start Algoze";
   song2.stop();
 } else {
   button2.innerText = "Stop Algoze";
   song2.loop();
 }
}
*/

//if song is playing, pan the audio 
function panning() {
 if(song.isLooping() && nose) {
   let panning = map(nose.x, 0, width, 1.0, -1.0); //pan right on 1 and left on -1
   song.pan(panning);
 } 
}

function delayAudio() {
 if(leftWrist && lwScore > 0.05) {     
     let delTime = map(leftWrist.y, 0, width, 4.0, 0.5);
     delay.delayTime(delTime);
     console.log("Delay Time: " + delTime);

     //delay.filter(20000);
     delay.amp(1.0);

 } 
 else {
   delay.delayTime(0.0);
   return;
 }
 song2.playMode('untilDone');

}

function reverbAudio() {
 if(rightWrist) {
   let dryWet = constrain(map(rightWrist.x, 0, width, 0, 1), 0, 1); 
   rev.drywet(dryWet);
 } else {
   rev.drywet(0);
 }
}

function keyPressed() {
 if(keyCode === LEFT_ARROW) {
   song2.play();
 }
}
2

It looks like an interesting project!

setInterval is to start a new “thread” separately from the draw loop. For example,

function draw() {
  setInterval(()=>console.log("hi"), 3000)
}

will end up with many messages on the console.

I’d recommend you to try to fix the problem without PoseNet because this logic is totally separate from PoseNet. For example, you can just use mouse position instead of the wrist. This will make you spot the problem easier, and plus, you will likely to have more comments as the code will be simpler without PoseNet. And once you figure out how to delay the sound, you can put it back to the original code.