Color picking in the openGL

Question

i've been trying to implement color picking and it just aint working right. the problem is that if initially paint my model in the different colors that are used for the picking (i mean, i give each triangle different color, which is his id color), it works fine (without texture or anything .. ), but if i put texture of the model, and that when the mouse is clicked i paint the model by giving each triangle a different color, it doesnt work.. here is the code:

public int selection(int x, int y) {        
    GL11.glDisable(GL11.GL_LIGHTING);
    GL11.glDisable(GL11.GL_TEXTURE_2D);

    IntBuffer viewport = BufferUtils.createIntBuffer(16); 
    ByteBuffer pixelbuff = BufferUtils.createByteBuffer(16);

    GL11.glGetInteger(GL11.GL_VIEWPORT, viewport);              

    this.render(this.mesh);

    GL11.glReadPixels(x, y, 1, 1, GL11.GL_RGB, GL11.GL_UNSIGNED_BYTE, pixelbuff);

    for (int m = 0; m < 3; m++)
        System.out.println(pixelbuff.get(m));

   GL11.glEnable(GL11.GL_TEXTURE_2D);
   GL11.glEnable(GL11.GL_LIGHTING);

    return 0;
}


public void render(GL_Mesh m, boolean inPickingMode)
{
    GLMaterial[] materials = m.materials;   // loaded from the .mtl file
    GLMaterial mtl;
    GL_Triangle t;
    int currMtl = -1;
    int i = 0;

    // draw all triangles in object
    for (i=0; i < m.triangles.length; ) {
        t = m.triangles[i];

        // activate new material and texture
        currMtl = t.materialID;
        mtl = (materials != null && materials.length>0 && currMtl >= 0)? materials[currMtl] : defaultMtl;
        mtl.apply();
        GL11.glBindTexture(GL11.GL_TEXTURE_2D, mtl.textureHandle);

        // draw triangles until material changes
        for ( ; i < m.triangles.length && (t=m.triangles[i])!=null && currMtl == t.materialID; i++) {
                drawTriangle(t, i, inPickingMode);

        }
    }
}

private void drawTriangle(GL_Triangle t, int i, boolean inPickingMode) {

    if (inPickingMode) {
        byte[] triColor = this.triangleToColor(i);

        GL11.glColor3ub((byte)triColor[2], (byte)triColor[1], (byte)triColor[0]);
    }

    GL11.glBegin(GL11.GL_TRIANGLES);

    GL11.glTexCoord2f(t.uvw1.x, t.uvw1.y);
    GL11.glNormal3f(t.norm1.x, t.norm1.y, t.norm1.z);
    GL11.glVertex3f( (float)t.p1.pos.x, (float)t.p1.pos.y, (float)t.p1.pos.z);

    GL11.glTexCoord2f(t.uvw2.x, t.uvw2.y);
    GL11.glNormal3f(t.norm2.x, t.norm2.y, t.norm2.z);
    GL11.glVertex3f( (float)t.p2.pos.x, (float)t.p2.pos.y, (float)t.p2.pos.z);

    GL11.glTexCoord2f(t.uvw3.x, t.uvw3.y);
    GL11.glNormal3f(t.norm3.x, t.norm3.y, t.norm3.z);
    GL11.glVertex3f( (float)t.p3.pos.x, (float)t.p3.pos.y, (float)t.p3.pos.z);

    GL11.glEnd();
}

as you can see, i have a selection function that's called everytime the mouse is clicked, i then disable the lightining and the texture, and then i render the scene again in the unique colors, and then read the pixles buffer, and the call of: GL11.glReadPixels(x, y, 1, 1, GL11.GL_RGB, GL11.GL_UNSIGNED_BYTE, pixelbuff); gives me wrong values .. and its driving me nutz ! btw, the main render function is render(mesh m, boolean inPickingMode) as u can see, you can also see that there is texture on the model before the mouse clicking ..

Answer 1

there are several problems with the example.

First, you're not clearing the color and depth-buffer when clicking the mouse (that causes the scene with color polygons to be mixed into the scene with textured polygons - and then it doesn't work). you need to call:

GL11.glClear(GL11.GL_COLOR_BUFFER_BIT | GL11.GL_DEPTH_BUFFER_BIT);

Second, it is probably a bad idea to use materials when color-picking. I'm not familiar with the GLMaterial class, but it might enable GL_COLOR_MATERIAL or some other stuff, which modifies the final color, even if lighting is disabled. Try this:

if(!inPickingMode) { // === add this line ===
    // activate new material and texture
    currMtl = t.materialID;
    mtl = (materials != null && materials.length>0 && currMtl >= 0)? materials[currMtl] : defaultMtl;
    mtl.apply();
    GL11.glBindTexture(GL11.GL_TEXTURE_2D, mtl.textureHandle);
} // === and this line ===

Next, and that is not related to color picking, you call glBegin() too often for no good reason. You can call it in render(), before the triangle drawing loop (but that shouldn't change how the result looks like):

GL11.glBegin(GL11.GL_TRIANGLES);
// draw triangles until material changes
for ( ; i < m.triangles.length && (t=m.triangles[i])!=null && currMtl == t.materialID; i++) {
        drawTriangle(t, i, inPickingMode);
}
GL11.glEnd();

--- now i am answering a little beyond the original question ---

The thing about color picking is, that the renderer has only limited number of bits to represent the colors (like as little as 5 bits per channel), so you need to use colors that do not have these bits set. It might be a bad idea to do this on a mobile device.

If your objects are simple enough (can be represented by, say a sphere, for picking), it might be a good idea to use raytracing for picking objects. It is pretty simple, the idea is that you take inverse of modelview-projection matrix, and transform points (mouse_x, mouse_y, -1) and (mouse_x, mouse_y, +1) by it, which will give you position of mouse at the near and at the far view plane, in object space. All you need to do is to subtract them to get direction of ray (origin is at the near plane), and you can pick your objects using this ray (google ray - sphere intersection).

float[] mvp = new float[16]; // this is your modelview-projection
float mouse_x, mouse_y; // those are mouse coordinates (in -1 to +1 range)
// inputs

float[] mvp_inverse = new float[16];
Matrix.invertM(mvp_inverse, 0, mvp, 0);
// inverse the matrix

float nearX = mvp_inverse[0 * 4 + 0] * mouse_x +
              mvp_inverse[1 * 4 + 0] * mouse_y +
              mvp_inverse[2 * 4 + 0] * -1 +
              mvp_inverse[3 * 4 + 0];
float nearY = mvp_inverse[0 * 4 + 1] * mouse_x +
              mvp_inverse[1 * 4 + 1] * mouse_y +
              mvp_inverse[2 * 4 + 1] * -1 +
              mvp_inverse[3 * 4 + 1];
float nearZ = mvp_inverse[0 * 4 + 2] * mouse_x +
              mvp_inverse[1 * 4 + 2] * mouse_y +
              mvp_inverse[2 * 4 + 2] * -1 +
              mvp_inverse[3 * 4 + 2];
float nearW = mvp_inverse[0 * 4 + 3] * mouse_x +
              mvp_inverse[1 * 4 + 3] * mouse_y +
              mvp_inverse[2 * 4 + 3] * -1 +
              mvp_inverse[3 * 4 + 3];
// transform the near point

nearX /= nearW;
nearY /= nearW;
nearZ /= nearW;
// dehomogenize the coordinate

float farX = mvp_inverse[0 * 4 + 0] * mouse_x +
             mvp_inverse[1 * 4 + 0] * mouse_y +
             mvp_inverse[2 * 4 + 0] * +1 +
             mvp_inverse[3 * 4 + 0];
float farY = mvp_inverse[0 * 4 + 1] * mouse_x +
             mvp_inverse[1 * 4 + 1] * mouse_y +
             mvp_inverse[2 * 4 + 1] * +1 +
             mvp_inverse[3 * 4 + 1];
float farZ = mvp_inverse[0 * 4 + 2] * mouse_x +
             mvp_inverse[1 * 4 + 2] * mouse_y +
             mvp_inverse[2 * 4 + 2] * +1 +
             mvp_inverse[3 * 4 + 2];
float farW = mvp_inverse[0 * 4 + 3] * mouse_x +
             mvp_inverse[1 * 4 + 3] * mouse_y +
             mvp_inverse[2 * 4 + 3] * +1 +
             mvp_inverse[3 * 4 + 3];
// transform the far point

farX /= farW;
farY /= farW;
farZ /= farW;
// dehomogenize the coordinate

float rayX = farX - nearX, rayY = farY - nearY, rayZ = farZ - nearZ;
// ray direction

float orgX = nearX, orgY = nearY, orgZ = nearZ;
// ray origin

And finally - a debugging suggestion: try to render with inPickingMode set to true so you can see what is it that you are actually drawing, on screen. If you see texture or lighting, then something went wrong.