Setting Pixels of a Webcam Capture

Hello everyone,

I’m working on a ‘pixel mask’ sketch which would take a Capture camera input, compare the color of pixels to a tracked color (retrieved by mousePressed()), then ‘mask’ those pixels by setting their alpha value to 0 thereby revealing the yellow background underneath. Eventually I would add various videos or other graphics in the background, but for now wanted to keep it simple. I adapted some code from a Coding Train tutorial which got me started, but I’ve run into several problems that have me at an impasse.

Here is a basic version of the sketch that simply sets the pixels of a given color to color(0,0) which theoretically should be transparent, revealing the yellow background. However, the alpha channel doesn’t seem to work. I read somewhere that you can do something like colorMode(ARGB) but this also didn’t reveal the background and was extremely unresponsive to other color changes.

The second, and perhaps even more concerning problem I ran into is that the pixel setting is super glitchy and flickering. Is my computer simply too slow to calculate all of the pixels at the given frameRate()? I am using a MacBook Pro 2018 15" with an i7 Intel Core, Radeon Pro 560X and 16GB Memory – not great but could be worse. I tried using bit shifting for colors instead of red() green() and blue() but this didn’t seem to make so much difference.

I also thought maybe it has to do with the order of execution, that the frames of the video are being redrawn too quickly, so I made a boolean which only redraws the video capture when the for() loop has finished calculating and setting the pixels’ color. This helped a bit but the problem remains.

As it stands now, the sketch runs, the pixels are set, but the background doesn’t shine through. It runs smoothly for a bit, then after 15 seconds or so becomes super choppy and I can see the yellow background flashing in bars, as if its being written over quickly, but the frame rate has dropped.

Maybe I’m simply going about this the wrong way, or there’s a way to optimize my code that I haven’t thought of yet. Any help would be greatly appreciated!


//pixel mask
//pixel calculations seems to work better when
//a part of capture event
//still a bit choppy

import processing.video.*;

Capture video;
Movie clip;

color trackColor;
float threshold = 50;
boolean loopEnd = true;

void setup() {

  size(1200, 800);
  frameRate(25);
  String[] cameras = Capture.list();
  printArray(cameras);
  video = new Capture(this, 1200, 800, cameras[0], 25);
  video.start();

  //clip = new Movie(this, "beach2.mp4");
  //clip.loop();
  //clip.volume(0);
  trackColor = color(255, 0, 0);
}


void draw() {

  //eventually implement movie in background
  //image(clip, 0, 0, width, height);
  //clip.loadPixels();

  //only draw when pixels have been calculated and set
  if (loopEnd) {
    //yellow background
    background(255, 255, 0);
    image(video, 0, 0);

    loopEnd = false;
  }
}

void captureEvent(Capture video) {

  video.read();

  video.loadPixels();

  //iterate through pixel array
  for (int x = 0; x < video.width; x++) {
    for (int y = 0; y < video.height; y++)
    {
      //location of individual pixels
      int loc = x + y * video.width;

      //what  is current color
      color currentColor = video.pixels[loc];

      //get color values
      int r1 = (currentColor >> 16) & 0xFF;  // Faster way of getting red(argb)
      int g1 = (currentColor >> 8) & 0xFF;   // Faster way of getting green(argb)
      int b1 = currentColor & 0xFF;//fast blue

      int r2 = (trackColor >> 16) & 0xFF;  // Faster way of getting red(argb)
      int g2 = (trackColor >> 8) & 0xFF;   // Faster way of getting green(argb)
      int b2 = trackColor & 0xFF; // Faster way of getting blue(argb)

      /*
      //alternate implementation to get color values
       float r1 = red(currentColor);
       float g1 = green(currentColor);
       float b1 = blue(currentColor);
       float r2 = red(trackColor);
       float g2 = green(trackColor);
       float b2 = blue(trackColor);
       */

      //compare similarity of current color to track color
      //using 3d distance function
      float d = distSq(r1, g1, b1, r2, g2, b2);

      //if distance is below threshold squared
      if ( d < threshold * threshold) {

        video.pixels[loc] = color(0, 0, 0, 0);
        //video.pixels[loc] = clip.pixels[loc];
      }
      if (x == video.width-1 && y == video.height-1) {
        //println("loop end");
        loopEnd = true;
      }
    }
  }
}

//3d distance for color compariso
float distSq(float x1, float y1, float z1, float x2, float y2, float z2) {
  float d = (x2-x1) * (x2-x1) + (y2 - y1 ) * (y2- y1) + (z2-z1) * (z2-z1);
  return d;
}

void mousePressed() {
  //save a color where the mouse is clicked in trackColor
  //number of pixels is pixelWidth * pixelHeight;
  // 1200 * 800 = 960,000
  int loc = mouseX + mouseY * video.width;
  trackColor = video.pixels[loc];
}

/*
void movieEvent(Movie clip) {
 clip.read();
 }
 */

Hello @dswayne92,

Modified your code:

// Modified code from here:
// https://discourse.processing.org/t/setting-pixels-of-a-webcam-capture/43976

import processing.video.*;

Capture video;
Movie clip;

color trackColor;
float threshold = 50;

void setup() 
  {
  size(640, 480);
  //frameRate(25);
  String[] cameras = Capture.list();
  printArray(cameras);
  video = new Capture(this, 640, 480);
  println(RGB);           // GLV added! 
  println(video.format);  // GLV added! 
  println(ARGB);          // GLV added! 
  video.format = ARGB;    // GLV added! 
  println(video.format);  // GLV added! 
  video.start();

  trackColor = color(255, 0, 0);
  noLoop();  // GLV added!
  }

void draw() 
  {
  background(255, 255, 0);
  video.format = 0;  // GLV added!          
  image(video, 0, 0);
  }

void captureEvent(Capture video) 
  {
  video.read();
  video.loadPixels();

  //iterate through pixel array
  for (int x = 0; x < video.width; x++) {
    for (int y = 0; y < video.height; y++)
    {
      //location of individual pixels
      int loc = x + y * video.width;

      //what  is current color
      color currentColor = video.pixels[loc];

      //get color values
      int r1 = (currentColor >> 16) & 0xFF;  // Faster way of getting red(argb)
      int g1 = (currentColor >> 8) & 0xFF;   // Faster way of getting green(argb)
      int b1 = currentColor & 0xFF;//fast blue

      int r2 = (trackColor >> 16) & 0xFF;  // Faster way of getting red(argb)
      int g2 = (trackColor >> 8) & 0xFF;   // Faster way of getting green(argb)
      int b2 = trackColor & 0xFF; // Faster way of getting blue(argb)

      //compare similarity of current color to track color
      //using 3d distance function
      float d = distSq(r1, g1, b1, r2, g2, b2);

      //if distance is below threshold squared
      if ( d < threshold * threshold) 
        {
        video.pixels[loc] = color(0, 0, 0, 0);
        }
      }
    }
  redraw(); // GLV added!
  }

//3d distance for color compariso
float distSq(float x1, float y1, float z1, float x2, float y2, float z2) {
  float d = (x2-x1) * (x2-x1) + (y2 - y1 ) * (y2- y1) + (z2-z1) * (z2-z1);
  return d;
}

void mousePressed() {
  //save a color where the mouse is clicked in trackColor
  //number of pixels is pixelWidth * pixelHeight;
  // 1200 * 800 = 960,000
  int loc = mouseX + mouseY * video.width;
  trackColor = video.pixels[loc];
}

Related discussions:

Scrutinize the changes and look up the references where needed.

:)

Hi @glv,

Brilliant! It would have taken me ages to stumble upon the format method. The alpha values are now set correctly. However, I’m still left with some flickering, especially if I increase the size of the window. I also tried adding a movie clip playing behind the webcam image and the flickering got worse. Is this a pure performance issue or is there something else I’m missing?

Additionally, I’m not sure I understand your use of noLoop() and redraw(). I thought these were primarily for stopping and updating the draw function? I tried removing them and it didn’t seem to make any obvious difference.

To save space I’ll just write code for the movie clip and where I placed it:

The clip is loaded in setup as is standard

  clip = new Movie(this, "beach2.mp4");
  clip.loop();
  clip.volume(0);

It’s read in a separate movieEvent function


void movieEvent(Movie clip) {
  clip.read();
}

And simply drawn to the sketch before the webcam video is


void draw()
{
  image(clip, 0, 0, width, height);
  //background(255, 255, 0);
  video.format = 0;  // GLV added!
  image(video, 0, 0);
}

Thanks very much for your advice!

1 Like

This was done so that I only updated after a successful captureEvent(); part of my debugging and development.

Another related topic:

References:

:)

Thanks again for the quick answer! The topic you linked was extremely helpful. It turns out the flickering was an issue with multiple threads being reordered before executing. A small research about the ‘volatile’ keyword in Java and looking at the code example posted yielded a quick fix.

I’m still experiencing some lag when using fullScreen() – thought the FX2D renderer seems to help – but now that the patch is working correctly I can take some time to do research about optimization on my own.

Here’s the code with the ‘volatile’ boolean added:


import processing.video.*;
import processing.javafx.*;

Capture video;
Movie clip;

color trackColor;
float threshold = 50;

//volatile booleans block processor reordering of thread execution
//shares values between threads
//any write to a volatile field happens before every read of the same field
volatile boolean drawing;

void setup()
{
  //fullScreen(FX2D);
  size(640, 480);
  //frameRate(25);
  String[] cameras = Capture.list();
  printArray(cameras);
  video = new Capture(this, width, height);
  println(RGB);
  println(video.format);
  println(ARGB);
  video.format = ARGB;
  println(video.format);

  clip = new Movie(this, "beach2.mp4");
  clip.loop();
  clip.volume(0);

  video.start();

  trackColor = color(255, 0, 0);
  noLoop();
}

void draw()
{
  drawing = true; //volatile boolean controls thread execution

  image(clip, 0, 0, width, height);
  //background(255, 255, 0);
  video.format = 0;
  image(video, 0, 0);

  drawing = false; //volatile boolean controls thread execution
}

void captureEvent(Capture video)
{
  //volatile boolean controls thread execution
  if (!drawing) {

    video.read();
    video.loadPixels();

    //iterate through pixel array
    for (int x = 0; x < video.width; x++) {
      for (int y = 0; y < video.height; y++)
      {
        //location of individual pixels
        int loc = x + y * video.width;

        //what  is current color
        color currentColor = video.pixels[loc];

        //get color values
        int r1 = (currentColor >> 16) & 0xFF;  // Faster way of getting red(argb)
        int g1 = (currentColor >> 8) & 0xFF;   // Faster way of getting green(argb)
        int b1 = currentColor & 0xFF;//fast blue

        int r2 = (trackColor >> 16) & 0xFF;  // Faster way of getting red(argb)
        int g2 = (trackColor >> 8) & 0xFF;   // Faster way of getting green(argb)
        int b2 = trackColor & 0xFF; // Faster way of getting blue(argb)

        //compare similarity of current color to track color
        //using 3d distance function
        float d = distSq(r1, g1, b1, r2, g2, b2);

        //play with threshold values. greater than vs less than
        //if distance is below threshold squared
        if ( d < threshold * threshold)
        {
          video.pixels[loc] = color(0, 0, 0, 0);
        }
      }
    }
    redraw();
  }
}


float distSq(float x1, float y1, float z1, float x2, float y2, float z2) {
  float d = (x2-x1) * (x2-x1) + (y2 - y1 ) * (y2- y1) + (z2-z1) * (z2-z1);
  return d;
}

void mousePressed() {

  int loc = mouseX + mouseY * video.width;
  trackColor = video.pixels[loc];
}

void movieEvent(Movie clip) {
  clip.read();
}
1 Like