Get mouseX and mouseY position with current transformation

Processing Coding Questions
1

How can I get the mouse position (and where z hits the camera plane), inside the transformation.
So I need this before the popMatrix().

void setup() {
  size(600, 600, P3D);
}


public void draw() {
    background(0);
    translate(width/2, height/2, -100);
    pushMatrix();
    float rot_y = map(mouseX, 0, width, 0, TWO_PI);
    rotateY(rot_y);
    int dim = min(width, height) / 2;
    noFill();
    stroke(255);
    box(dim);
    
    // now what is the mouseX, mouseY position (and z, where it hits the near plane of the camera)
    // with the current transformation?
    // float x = ???;
    // float y = ???;
    // float z = ???;
   
    popMatrix();
}

I tried things with getting the matrix and inverting it and multiplying the mouse coordinates with it, but no luck.

2

I offer screenX and screenY which gives you the position of a 3D rotated point on the 2D screen so that you can check mouseX,mouseY against it

(modelX, modelY, modelZ does sth. similar)

3

No, it’s more complicated.
What I want is equal to unity’s ScreenToWorldPoint

I made 2 attempts, I think I’m close, but this is above my head:

PVector screenPointToWorld(PVector sp) {
    PGraphicsOpenGL _g = ((PGraphicsOpenGL)g);

    // cam.worldToCameraMatrix?? Not sure about this one
    PMatrix3D worldToCameraMatrix = new PMatrix3D(_g.camera);
    worldToCameraMatrix.invert();

    PMatrix3D world2Screen = new PMatrix3D(_g.projection);
    world2Screen.apply(worldToCameraMatrix);
    
    PMatrix3D screen2World = new PMatrix3D(world2Screen);
    screen2World.invert();

    float[] inn = new float[4];

    inn[0] = 2.0f * (sp.x / width) - 1.0f;
    inn[1] = 2.0f * (sp.y / height) - 1.0f;
    inn[2] = _g.cameraNear;
    inn[3] = 1.0f;  

    float[] pos = new float[4];
    screen2World.mult(inn, pos);

    int X = 0;
    int Y = 1;
    int Z = 2;
    int W = 3;

    pos[W] = 1.0f / pos[W];

    pos[X] *= pos[W];
    pos[Y] *= pos[W];
    pos[Z] *= pos[W];

    return new PVector(pos[X], pos[Y], pos[Z]);
}


PVector screenPointToWorld(PVector sp) {
    PGraphicsOpenGL _g = ((PGraphicsOpenGL)g);

    // cam.worldToCameraMatrix?? Not sure about this one
    PMatrix3D final_matrix = new PMatrix3D(_g.modelview);
    final_matrix.apply(_g.projection);
    final_matrix.invert();

    float[] in = new float[4];

    in[0] = sp.x;
    in[1] = sp.y;
    in[2] = sp.z;
    in[3] = 1.0f;  

    /* Map x and y from window coordinates */
    // in[0] = (in[0] - viewport[0]) / viewport[2];
    // in[1] = (in[1] - viewport[1]) / viewport[3];
    in[0] = in[0] / width;
    in[1] = in[1] / height;

    /* Map to range -1 to 1 */
    in[0] = in[0] * 2 - 1;
    in[1] = in[1] * 2 - 1;
    in[2] = in[2] * 2 - 1;

    float[] out = new float[4];

    final_matrix.mult(in, out);

    out[0] /= out[3];
    out[1] /= out[3];
    out[2] /= out[3];
    
    return new PVector(out[X], out[Y], out[Z]);
}
1 Like