How to combine mouse clicks in 3D and processing to be able to click on items in 3D to switch interfaces

Processing Coding Questions
1

How to combine mouse clicks in 3D and processing to be able to click on items in 3D to switch interfaces,
I need to do a 3D scan of the room and can click on the items in the room, but I don’t know how to use mouse clicks and 3D scans together? Or does this approach work?

1 Like
2

Hello!

And welcome to the forum! Great to have you here.

Can you provide more details? In which format are your 3D scans?

Are the items separate data items in a data list? Because otherwise you need a image recognition (in 3D)?

Anyways, here is an quick example when you know the rects data in a 3D grid you can use screenX,screenY to match with the mouse. See reference screenX(),screenY().

It’s all in the class:

  • Basically, during display the rects in 3D, we store the screen position using screenX(),screenY() into scX, scY for every rect.
  • When the mouse gets pressed, we check mouseX and mouseY against scX, scY.
  • When they are close, you gotto match.

Warm regards,

Chrisir


Cell[][][] grid = new Cell[6][6][6];

void setup() {
  size(1900, 800, P3D);

  int groesseZelle=44;

  for (int x=0; x < grid.length; x++) {
    for (int y=0; y < grid.length; y++) {
      for (int z=0; z < grid.length; z++) {
        PVector pos=new PVector(width/2+x*groesseZelle-3*groesseZelle,
          height/2+ y*groesseZelle-3*groesseZelle,
          z*groesseZelle+222);
        // Cell erstellen
        grid[x][y][z] = new Cell( pos, ' ', color(x*6+10) );
      }
    }
  }

  //
} // func

void draw() {
  background(0);


  for (int x=0; x < grid.length; x++) {
    for (int y=0; y < grid.length; y++) {
      for (int z=0; z < grid.length; z++) {
        grid[x][y][z].display();
      }
    }
  }
}

// ---------------------------------------------------------------------------------

void mousePressed() {

  // Check the Scrabble grid

  for (int x=0; x < grid.length; x++) {
    for (int y=0; y < grid.length; y++) {
      for (int z=0; z < grid.length; z++) {

        // Wenn Maus
        if ( grid[x][y][z].mouseOver()) {
          // toggle
          grid[x][y][z].on=
            ! grid[x][y][z].on;
          // Leave
          return;
        }
      }
    }
  }
}

// ---------------------------------------------------------------------------------

// colors

final color BLACK  = color(0);
final color WHITE  = color(255);

final color RED    = color(255, 0, 0);
final color GREEN  = color(0, 255, 0);
final color BLUE   = color(0, 0, 255);

final color PINK       = color(255, 0, 255);
final color YELLOW     = (#FFF80A);
final color YELLOW2    = color(255, 255, 0);
final color TURQUOISE  = color(0, 255, 255);

final color BROWN      = #B9780D;


//=====================================================================
// class
//

class Cell {

  PVector pos;
  color   cellColor;

  int groesseZelle=44;

  float scX, scY;

  boolean on = false;

  // --------------------------------------------------

  // constr
  Cell (PVector pos_,
    char text_,
    color col_) {
    //
    pos   = pos_.copy();
    cellColor = col_;
    //
  }// constr

  // --------------------------------------------------

  void display() {
    // Anzeige

    pushMatrix();
    translate(pos.x, pos.y, pos.z);

    stroke(220);
    noFill();

    rectMode(CENTER);
    pushMatrix();
    translate(0, 30/2, 0);
    rotateX(radians(90));
    fill(255);
    if (on)
      fill(RED);
    rect(0, 0, groesseZelle/2, groesseZelle/2);
    scX = screenX(0, 0, 0);
    scY = screenY(0, 0, 0);
    popMatrix();

    popMatrix();
  } // method

  boolean mouseOver() {
    return
      dist (mouseX, mouseY,
      scX, scY)<19;
  }
  //
} //class
//
3

One way to pick objects in 3-D is to render the scene with no lighting and no anti-aliasing (noSmooth()) and with the color set to the id of each object. You can then read out the color value of the pixel under the mouse position to find the id of the object that is picked.

int N = 2000;

class Thing {
  float x, y, z;
  color clr;
  int id;
  
  Thing( int id_ ) {
    z = random( -1, 1 );
    float r = pow( random(1), 1.0/3 );
    float rxy = r * sqrt( 1 - z*z );
    float a = random(TAU);
    x = rxy * cos(a);
    y = rxy * sin(a);
    z *= r;
    clr = color( random(64, 192), random(64, 192), random(64, 192) );
    id = id_;
  }
  
  void drawGeom( PGraphics pg ) {
    pg.push();
    pg.translate( x, y, z );
    pg.rotateX( id );
    pg.rotateY( id );
    pg.box( 0.06, 0.06, 0.06 );
    pg.pop();
  }
  
  void draw( PGraphics pg ) {
    if( id == picked ) pg.fill( 255 );  else pg.fill( clr );
    drawGeom( pg );
  }
  
  void drawID( PGraphics pg ) {
    pg.fill( id>>16, (id>>8)&0xff, id&0xff );
    drawGeom( pg );
  }
}

Thing[] things;

PGraphics scene, picker;
int picked;

void setup() {
  size( 900, 900, P3D );
  noStroke();
  things = new Thing[N];
  for( int i=0; i<N; i++ ) things[i] = new Thing( i );

  scene = createGraphics( width, height, P3D );
  picker = createGraphics( width, height, P3D );
  picker.noSmooth();
  picked = -1;
}

void draw() {
  float t = frameCount * 0.001;
  picker.beginDraw();
  picker.camera( 0, 0, 3,  0, 0, 0,  0, 1, 0 );
  picker.perspective( PI/4, 1.0*width/height, 0.1, 10 );
  picker.background(255);
  picker.noStroke();
  picker.noLights();
  picker.rotateY( t );
  for( int i=0; i<N; i++ ) things[i].drawID( picker );
  picker.endDraw();
  picker.loadPixels();
  picked = picker.pixels[ mouseY*width+mouseX ] & 0xffffff;

  scene.beginDraw();
  scene.camera( 0, 0, 3,  0, 0, 0,  0, 1, 0 );
  scene.perspective( PI/4, 1.0*width/height, 0.1, 10 );
  scene.background(0);
  scene.lights();
  scene.noStroke();
  scene.rotateY( t );
  for( int i=0; i<N; i++ ) things[i].draw( scene );
  scene.endDraw();
  image( scene, 0, 0 );
}
1 Like