Kinect PV2 record color point cloud

#1

Hello, i am trying to record the point cloud with color using this Kinect PV2 library,
apparenty the getPointCloudColorPos() returns - INFINITY values, so i used 3 buffers two for the represantation in the processing sketch, and then took the getColorChannelBuffer() and .getPointCloudDepthPos() to take the data and record them in a ply file. I made it as soon as takes the data to write them in a ply file, but takes ages from every frame.

So i used the example of RecordPointCloud and add two floatbuffers in the class and then record them.

Everything goes well, until i press record that gives me :

Version: 0.7.8
#
# A fatal error has been detected by the Java Runtime Environment:
#
#  EXCEPTION_ACCESS_VIOLATION (0xc0000005) at pc=0x00000000778bd6c1, pid=25044, tid=0x0000000000006510
#
# JRE version: Java(TM) SE Runtime Environment (8.0_144-b01) (build 1.8.0_144-b01)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.144-b01 mixed mode windows-amd64 compressed oops)
# Problematic frame:
# V  [jvm.dll+0x15d6c1]
#
# Failed to write core dump. Minidumps are not enabled by default on client versions of Windows
#
# An error report file with more information is saved as:
# E:\Pool\processing-3.3.6\hs_err_pid25044.log
#
# If you would like to submit a bug report, please visit:
#   http://bugreport.java.com/bugreport/crash.jsp

Here is the code:

/*
Thomas Sanchez Lengeling.
 http://codigogenerativo.com/
 
 KinectPV2, Kinect for Windows v2 library for processing
 
 Point Cloud Color Example using low level openGL calls and Shaders
 */

import java.nio.*;

import KinectPV2.*;

private KinectPV2 kinect;

//values for the 3d scene
//rotation
float a = 3;
//z scale value
int   zval     = 350;
//value to scale the point cloud
float scaleVal = 990;

//rotation
float rotY = 0;
float rotZ = 0;
float rotX = PI;

float depthVal = 0;

int vertLoc;
int colorLoc;


int numFrames =  100; // 30 frames  = 1s of recording
int frameCounter = 0; // frame counter

boolean recordFrame = false;  //recording flag
boolean doneRecording = false;

ArrayList<FrameBuffer> mFrames;
//openGL instances
//render ot openGL object
PGL pgl;

//create a shader
PShader sh;


//VBO buffer location in the GPU
int vertexVboId;
int colorVboId;
int colorVDepth;

public void setup() {
  size(1280, 768, P3D);

  kinect = new KinectPV2(this);
  
  mFrames = new ArrayList<FrameBuffer>();

  kinect.enableDepthImg(true);
  kinect.enableColorImg(true);
  kinect.enableColorPointCloud(true);
  kinect.enablePointCloud(true);

  kinect.init();

  //create shader object with a vertex shader and a fragment shader
  sh = loadShader("frag.glsl", "vert.glsl");


  //create VBO

  PGL pgl = beginPGL();

  // allocate buffer big enough to get all VBO ids back
  IntBuffer intBuffer = IntBuffer.allocate(3);
  pgl.genBuffers(3, intBuffer);

  //memory location of the VBO
  vertexVboId = intBuffer.get(0);
  colorVboId = intBuffer.get(1);
  colorVDepth = intBuffer.get(2);

  endPGL();
}

public void draw() {



  background(0);

  //image(kinect.getColorImage(), 0, 0, 320, 240);


  // The geometric transformations will be automatically passed to the shader
  pushMatrix();
  translate(width / 2, height / 2, zval);
  scale(scaleVal, -1 * scaleVal, scaleVal);
  rotate(a, 0.0f, 1.0f, 0.0f);

  //render to the openGL object
  pgl = beginPGL();
  sh.bind();


  //obtain the point cloud positions
  FloatBuffer pointCloudBuffer = kinect.getPointCloudColorPos();

  //get the color for each point of the cloud Points
  FloatBuffer colorBuffer      = kinect.getColorChannelBuffer();
  
  FloatBuffer DepthBuffer = kinect.getPointCloudDepthPos();
  
////DEBUG
//  for (int i = 0; i < 5000; i++) {
//  println("r :"+colorBuffer.get(i*3 + 0));
//  println("g :"+colorBuffer.get(i*3 + 1));
//  println("b :"+colorBuffer.get(i*3 + 2));
//    println("x :"+DepthBuffer.get(i*3 + 0));
//  println("y :"+DepthBuffer.get(i*3 + 1));
//  println("z :"+DepthBuffer.get(i*3 + 2));
//}

  //send the the vertex positions (point cloud) and color down the render pipeline
  //positions are render in the vertex shader, and color in the fragment shader
  vertLoc = pgl.getAttribLocation(sh.glProgram, "vertex");
  pgl.enableVertexAttribArray(vertLoc);
  
  
  //enable drawing to the vertex and color buffer
  colorLoc = pgl.getAttribLocation(sh.glProgram, "color");
  pgl.enableVertexAttribArray(colorLoc);

  int vertData = kinect.WIDTHColor * kinect.HEIGHTColor * 3;

  //vertex
  {
    pgl.bindBuffer(PGL.ARRAY_BUFFER, vertexVboId);
    // fill VBO with data
    pgl.bufferData(PGL.ARRAY_BUFFER, Float.BYTES * vertData, pointCloudBuffer, PGL.DYNAMIC_DRAW);
    // associate currently bound VBO with shader attribute
    pgl.vertexAttribPointer(vertLoc, 3, PGL.FLOAT, false, Float.BYTES * 3, 0);
  }

  // color
  {
    pgl.bindBuffer(PGL.ARRAY_BUFFER, colorVboId);
    // fill VBO with data
    pgl.bufferData(PGL.ARRAY_BUFFER, Float.BYTES * vertData, colorBuffer, PGL.DYNAMIC_DRAW);
    // associate currently bound VBO with shader attribute
    pgl.vertexAttribPointer(colorLoc, 3, PGL.FLOAT, false, Float.BYTES * 3, 0);
  }

  // unbind VBOs
  pgl.bindBuffer(PGL.ARRAY_BUFFER, 0);

  //draw the point cloud as a set of points
  pgl.drawArrays(PGL.POINTS, 0, vertData);

  //disable drawing
  pgl.disableVertexAttribArray(vertLoc);
  pgl.disableVertexAttribArray(colorLoc);

  //close the shader
  sh.unbind();
  //close the openGL object
  endPGL();

allocateFrame(colorBuffer, DepthBuffer );

  writeFrames();

  popMatrix();

  stroke(255, 0, 0);
  text(frameRate, 50, height- 50);
}


void allocateFrame(FloatBuffer buffer1, FloatBuffer buffer2) {
  if (recordFrame) {
    if ( frameCounter < numFrames) {
      FrameBuffer frameBuffer = new FrameBuffer(buffer1, buffer2);
      frameBuffer.setFrameId(frameCounter);

      mFrames.add(frameBuffer);
    } else {
      recordFrame = false;
      doneRecording = true;
    }
    frameCounter++;
  }
}

void writeFrames() {
  if (doneRecording) {
    for (int i = 0; i < mFrames.size(); i++) {
      FrameBuffer fBuffer =  (FrameBuffer)mFrames.get(i);
      fBuffer.saveOBJFrame();
    }
    doneRecording = false;
    println("Done Recording frames: "+numFrames);
  }
}


public void mousePressed() {

  println(frameRate);
  //  saveFrame();
}

public void keyPressed() {
  
    if (key == 'r') {
    recordFrame = true;
  }
  
  if (key == 'a') {
    zval +=1;
    println(zval);
  }
  if (key == 's') {
    zval -= 1;
    println(zval);
  }

  if (key == 'z') {
    scaleVal += 1;
    println(scaleVal);
  }
  if (key == 'x') {
    scaleVal -= 1;
    println(scaleVal);
  }

  if (key == 'q') {
    a += 0.1;
    println("angle "+a);
  }
  if (key == 'w') {
    a -= 0.1;
    println("angle "+a);
  }

  if (key == 'c') {
    depthVal -= 0.01;
    println(depthVal);
  }

  if (key == 'v') {
    depthVal += 0.01;
    println(depthVal);
  }
}

And here is the class:

/*
Simple class that manager saving each FloatBuffer and writes the data into a OBJ file
*/
class FrameBuffer {

  FloatBuffer frameColor;
  FloatBuffer frame;
  
  //id of the frame
  int frameId;

  FrameBuffer(FloatBuffer f, FloatBuffer fcolor) {
    frame = clone(f);
    frameColor = clone(fcolor);
  }
  

  void setFrameId(int fId) {
    frameId = fId;
  }

  /*
  Writing of the obj file,
  */
  void saveOBJFrame() {
    int vertData = 512 * 424;
    String[] points = new String[vertData+8];

    //Iterate through all the XYZ points
    for (int i = 0; i < vertData; i++) {
      ////DEBUG
//  for (int i = 0; i < 5000; i++) {
  println("r :"+frameColor.get(i*3 + 0));
  println("g :"+frameColor.get(i*3 + 1));
  println("b :"+frameColor.get(i*3 + 2));
    println("x :"+frame.get(i*3 + 0));
  println("y :"+frame.get(i*3 + 1));
  println("z :"+frame.get(i*3 + 2));
//}
      
      
      
      
      //float x =  frame.get(i*3 + 0);
      //float y =  frame.get(i*3 + 1);
      //float z =  frame.get(i*3 + 2);
      //points[0]="ply";
      //points[1]="element vertex "+ vertData;
      //points[2]="property float x";
      //points[3]="property float y";
      //points[4]="property float z";
      //points[5]="element face 0";
      //points[6]="property list uint int vertex_indices";
      //points[7]="end_header";
      //points[8+i] = x+" "+y+" "+z;
    }
    
    //saveStrings("data/"+frameId+".ply", points);
    //println("Done Saving Frame "+frameId);
  }

  //Simple function that copys the FloatBuffer to another FloatBuffer
  public  FloatBuffer clone(FloatBuffer original) {
    FloatBuffer clone = FloatBuffer.allocate(original.capacity());
    original.rewind();//copy from the beginning
    clone.put(original);
    original.rewind();
    clone.flip();
    return clone;
  }
  
}

Do you have any idea, what might be?

#2

Can you explain this statement? Do you mean never returns, or it returns a… actually, I’m just not sure what this means. What was the behavior that led you to conclude this?

#4

rather than values it console.logs “-INFINITY”.

#5

Mysterious. I’m not sure where that INFINITY message is coming from, but I don’t think that keyword appears in that library.

Here is the source search for “getPointCloudColorPos” to perhaps consider inspecting for clues to what is happening…

#6

with adding this for loop to see the data of the buffers at line 129:

for (int i = 0; i < 5000; i++) {
  println("r :"+colorBuffer.get(i*3 + 0));
  println("g :"+colorBuffer.get(i*3 + 1));
  println("b :"+colorBuffer.get(i*3 + 2));
    println("x_c :"+pointCloudColorBuffer.get(1*3+0));
  println("y_c :"+pointCloudColorBuffer.get(i*3 + 1));
  println("z_c :"+pointCloudColorBuffer.get(i*3 + 2));
      println("x_p :"+pointCloudBuffer.get(1*3+0));
  println("y_p :"+pointCloudBuffer.get(i*3 + 1));
  println("z_p :"+pointCloudBuffer.get(i*3 + 2));
}

it console.logs :

r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0
r :254.0
g :254.0
b :254.0
x_c :-Infinity
y_c :-Infinity
z_c :-Infinity
x_p :0.0
y_p :0.0
z_p :0.0

I think maybe is a float infinity value.