P5 Delay Sound without User Interaction

p5.js Libraries
1

Hi! I’m working on a project involving Posenet where body movement triggers sound processing. So far I have the nose panning audio from left to right and the left wrist’s vertical motion causing a delay on a bell sound. However, the bell sound file keeps continuously playing and I can’t figure out where to call song2.play() in my code to have it play the bell once and automatically be delayed by the left wrist’s vertical position. Thoughts?

let video;
let poseNet;
let poses = [];
let skeletons = [];
let nose, rightWrist, leftWrist, rightKnee, leftKnee;
let nScore, rwScore, lwScore, rkScore, lkScore;
let song;
let button;
let fft, noise, fil;
let delay, rev;

function preload() {
  soundFormats('mp3', 'ogg');
  song = loadSound('dhol_longer.mp3');
  song2 = loadSound('bell.mp3');
}

function modelLoaded() {
  console.log("Model Loaded");
}

function setup() {
  cnv = createCanvas(640, 480);
  video = createCapture(VIDEO);
  video.size(width, height);

  button = document.getElementById("start");
  button.addEventListener("click", handleStartButtonClick);

  poseNet = ml5.poseNet(video, modelLoaded);
  poseNet.on('pose', getPoses);
  video.hide();

  delay = new p5.Delay();
  delay.process(song2, 0.12, 0.5, 2300);
  delay.setType('pingPong');

  rev = new p5.Reverb();
  song.disconnect();
  rev.process(song, 3, 2);
}

  function draw() {
    //mirrors image so you're viewing 
    translate(video.width, 0);
    scale(-1, 1);
    image(video, 0, 0, video.width, video.height);
  
    //draws dots on nose, wrists, and knees 
    drawPoints();

    //delays and filters audio with wrist movement
    if(leftWrist && (320 < leftWrist.x < 640)) {
      delayAudio();
    } else {
      return;
    }

    //reverb and then panning
    reverbAudio();

    //pans audio by following nose movement 
    panning();

    //delay into reverb into panning

    //cut out all other audio while triggering one sound
    //might be smarter to use quadrants and body part positioning 
    //
  }

function getPoses(poses) {
  if(poses.length > 0) {
    let pose = poses[0].pose;

    nose = pose.keypoints[0].position;
    nScore = pose.keypoints[0].score;

    leftWrist = pose.keypoints[9].position;
    lwScore = pose.keypoints[9].score;

    rightWrist = pose.keypoints[10].position;
    rwScore = pose.keypoints[10].score;

    leftKnee = pose.keypoints[13].position;
    lkScore = pose.keypoints[13].score;

    rightKnee = pose.keypoints[14].position;
    rkScore = pose.keypoints[14].score;
  }
}

function drawPoints() {
  if(nose && nScore > 0.6) {
    stroke(255, 0, 0);
    strokeWeight(15);
    point(nose.x, nose.y);
  }

  if(rightWrist && rwScore > 0.2) {
    stroke(0, 0, 255);
    strokeWeight(15);
    point(rightWrist.x, rightWrist.y); 
  }

  if(leftWrist && lwScore > 0.2) {
    stroke(0, 255, 255);
    strokeWeight(15);
    point(leftWrist.x, leftWrist.y);
  }

  if((rightKnee && leftKnee) && (rkScore && lkScore > 0.6)) {
    stroke(0, 255, 0);
    strokeWeight(15);
    point(rightKnee.x, rightKnee.y);
    stroke(0, 255, 0);
    strokeWeight(15);
    point(leftKnee.x, leftKnee.y);
  }
}

function handleStartButtonClick() {
  if(song.isLooping()) {
    button.innerText = "Start Sound";
    song.stop();
  } else {
    button.innerText = "Stop Sound";
    song.loop();
  }
}

function panning() {
  if(song.isLooping() && nose) {
    let panning = map(nose.x, 0, width, 1.0, -1.0); //pan right on 1 and left on -1
    song.pan(panning);
  } 
}

/**
 * if song isn't playing, do the delay code otherwise don't collect delay data
 * 
 * I tried putting song2.play() in all three spots in the function below and 
 * it still was playing the bell continuously
 */
function delayAudio() {
  if(leftWrist) {
    if(!song2.isPlaying()) {
      let delTime = map(leftWrist.y, 0, width, 1.0, 0.01);
      delay.delayTime(delTime);

      song2.playMode('untilDone');
      //song2.play();
    } 
    song2.play();

  } else {
    return;
  }
  //song2.play();  
}

function reverbAudio() {
  let dryWet = constrain(map(mouseX, 0, width, 0, 1), 0, 1);
  rev.drywet(dryWet);
}

function mousePressed() {
  song2.play();
}
2

hi! I cannot directly find a solution, but I suggest narrowing down the problem by removing posenet for now. For example, what if you replace the tracking with mousePress to trigger the delay? Would it make the same problem? If so, it has nothing to do with posenet and if not, it may be related to posenet. That will make it easier for others to test and debug, and also helps you making the debug process fast. Also if you can upload the sketch to the web editor and share the link, that would be great!