Picking in OpenGL using glRenderMode(GL_SELECT) and glReadPixels

Question

I'm trying to do selection in opengl but it is not working. I draw objects I receive from a .obj file (v, vn, f, o and such indices). Each object consists from "groups" and each group is a group of GL_POLYGON. Here is the draw function:

void draw(GLenum mode) {
    glBegin(GL_LINES);
    glColor3f(1, 0, 0);
    glVertex3f(0.0, 0.0, 0.0);
    glVertex3f(100.0, 0.0, 0.0);

    glColor3f(0, 0, 1);
    glVertex3f(0.0, 0.0, 0.0);
    glVertex3f(0.0, 100.0, 0.0);

    glColor3f(0, 1, 0);
    glVertex3f(0.0, 0.0, 0.0);
    glVertex3f(0.0, 0.0, 100.0);
    glEnd();

    glColor4f(1.0, 1.0, 1.0, 1.0);

    if (changeFOV) {
        fovAngle += fovScale;
        changeFOV = false;
        setTransformations();
    }
    for (unsigned int i = 0; i < objects.size(); i++) {
        objectItem currObject = objects[i];
        for (unsigned int j = 0; j < currObject.getGroups().size(); j++) {
            group currGroup = currObject.getGroups().at(j);
            for (unsigned int k = 0; k < currGroup.getFs().size(); k++) {
                if (mode == GL_SELECT)
                    glPushName(currGroup.getName());
                glPushMatrix();
                vector<pair<int, int> > currF = currGroup.getFs()[k];
                glBegin(GL_POLYGON);
                for (unsigned int kk = 0; kk < currF.size(); kk++) {
                    Vector3f currVertex = vertexes.at(
                            (currF.at(kk).first - 1 >= 0) ?
                                    currF.at(kk).first - 1 : 0);

                    Vector3f currNormal = vertexesNormal.at(
                            (currF.at(kk).second - 1 >= 0) ?
                                    currF.at(kk).second - 1 : 0);

                    glNormal3f(currNormal.x, currNormal.y, currNormal.z);
                    glVertex3f(currVertex.x / 1, currVertex.y / 1,
                            currVertex.z / 1);
                }
                glEnd();
                glPopMatrix();
            }
        }
    }
}

The drawing works ok and I see the object on the screen.

This is all the picking procedure

/*      PICKING     */
void processHits(GLint hits, GLuint *buffer) {
    float z1, z2;
    for (int i = 0; buffer[i] > 0; i += 5) {
        z1 = buffer[i + 1] / 4294967295.0;
        z2 = buffer[i + 2] / 4294967295.0;
        printf("z1 = %f ,z2 = %f zValue = %f\n", z1, z2, zValue[0]);
        if ((zValue[0] <= z1 + 0.0001 && zValue[0] >= z2 - 0.0001)
                || (zValue[0] >= z1 - 0.0001 && zValue[0] <= z2 + 0.0001)) { //try to locate which name is correlated with the pressed pixel according to z value
            ii = buffer[i + 3];
            jj = buffer[i + 4];
        }

    }
}

void startPicking(GLuint *selectionBuf) {
    glSelectBuffer(bufSize, selectionBuf); //declare buffer for input in selection mode
    glRenderMode(GL_SELECT); //change to selecting mode
    glInitNames();          //initialize names stack
    glPushName(-1);         //push name
}

void pick(int button, int x, int y) {
    //use selection mode to pick
    glReadPixels(x, viewport[3] - y, 1, 1, GL_RGBA, GL_FLOAT, pix);
    //printf("depth = %f, x = %d, y = %d\n",pixels[(viewport[3]-y)*512+x],x,viewport[3]-y);
    glMatrixMode(GL_PROJECTION);
    glReadPixels((GLdouble) x, (GLdouble) viewport[3] - y, 2, 2,
            GL_DEPTH_COMPONENT, GL_FLOAT, zValue);
    glPushMatrix(); //saves current projection matrix
    startPicking(selectionBuf); //preper selection mode
    glLoadIdentity();
    gluPickMatrix((GLdouble) x, (GLdouble) viewport[3] - y, 1, 1, viewport); //change matrices so only the area of the picking pixel can be seen.
    gluPerspective(fovAngle, 1, near, far); //return to perspective state
    glMatrixMode(GL_MODELVIEW);
    draw(GL_SELECT); //draws board on background
    hits = glRenderMode(GL_RENDER); //gets hits number
    glMatrixMode(GL_PROJECTION);
    glPopMatrix(); //restores projection matrix
    glMatrixMode(GL_MODELVIEW);
    processHits(hits, selectionBuf); //check hits
    if(hits > 0)
        printf("touched: %d\n",selectionBuf[3]);
    //printf("depth %f hits: %d\n\n",pixels[(viewport[3]-y)*512+x], hits);
    if (zValue[0] < 1.0) {
        isPick = true;
        xx = x;
        yy = y;
        if (button == GLUT_RIGHT_BUTTON)
            zMove = true;
        else
            zMove = false;
    }
}

the pick function is called when the mouse is clicked (using opengl mouse function). The error I'm receiving is that no objects appears to be hit when clicking on an object.

I'm using Ubuntu 14.04 LTS with Opengl 3.0

I don't know how to ask or what specifically ask, I would appreciate some inputs on the code if you see something wrong..

Answer 1

You seemed to have missed using glPopName()

GLNames used in the selection buffer are pushed on a stack. So unless you call glPopName() the stack would never unwind. The working of this is similar to the calls glPushMatrix() and glPopMatrix().

Typically this is what the code flow looks like.

//Push the name of the primitives on top of selection stack
glPushName(...)
  //Set Transformations / Draw the primitives
  ..
  ..
//Pop the name (Clear the stack for pushing another name)
glPopName();