Trackball issue with Raypicking

Question

I'm trying to develop a 3D raypicking in my OpenGL scene.

I have a working OBJ loader with a trackball.

char*      model_file = NULL;       /* name of the obect file */
GLuint     model_list = 0;      /* display list for object */
GLMmodel*  model;               /* glm model data structure */
GLfloat    scale;               /* original scale factor */
GLfloat    smoothing_angle = 90.0;  /* smoothing angle */
GLfloat    weld_distance = 0.00001; /* epsilon for welding vertices */
GLboolean  facet_normal = GL_FALSE; /* draw with facet normal? */
GLboolean  bounding_box = GL_FALSE; /* bounding box on? */
GLboolean  spheres = GL_FALSE;
GLboolean  performance = GL_FALSE;  /* performance counter on? */
GLboolean  stats = GL_FALSE;        /* statistics on? */
GLuint     material_mode = 0;       /* 0=none, 1=color, 2=material, 3=texture */
GLint      entries = 0;         /* entries in model menu */
GLdouble   pan_x = 0.0;
GLdouble   pan_y = 0.0;
GLdouble   pan_z = 0.0;
char texnames[1][64] = {"foto_rgb.ppm"};
//char texnames[1][64] = {"grid.ppm"};
GLint w,h;
GLubyte* texture;

void convert2dto3D(int mouseX, int mouseY){
    double matModelView[16], matProjection[16]; 
    int viewport[4]; 

    glGetDoublev( GL_MODELVIEW_MATRIX, matModelView ); 
    glGetDoublev( GL_PROJECTION_MATRIX, matProjection ); 
    glGetIntegerv( GL_VIEWPORT, viewport );


    double winX = (double)mouseX; 
    double winY = viewport[3] - (double)mouseY; 

    gluUnProject(winX, winY, 0.0, matModelView, matProjection, 
                 viewport, &m_start.x, &m_start.y, &m_start.z); 

    gluUnProject(winX, winY, 1.0, matModelView, matProjection, 
                 viewport, &m_end.x, &m_end.y, &m_end.z);

    //printf("Coordinata 2D %d %d\nCoordinata camera far: %f %f %f\n", mouseX, mouseY, m_end.x, m_end.y, m_end.z);
    //printf("Coordinate camera near: %f %f %f\n", m_start.x, m_start.y, m_start.z);
    //fflush(stdout);

    mouseClicked = GL_TRUE;

    Vector3d nearPoint(m_start.x, m_start.y, m_start.z);
    Vector3d farPoint(m_end.x, m_end.y, m_end.z);

    Vector3d ray = farPoint - nearPoint;

}

void line (void) {

    glLineWidth(10);
    //glPointSize(50.2);
    glColor3f(0.0f, 1.0f, 0.0f);      
    glBegin(GL_LINES);
        glVertex3f( m_start.x, m_start.y, m_start.z );
        glVertex3f( m_end.x, m_end.y, m_end.z );
    glEnd();
    glColor3f(1.0f,1.0f,1.0f);
    glLineWidth(1);

}


void lists(void){
    GLfloat ambient[] = { 0.2, 0.2, 0.2, 1.0 };
    GLfloat diffuse[] = { 0.8, 0.8, 0.8, 1.0 };
    GLfloat specular[] = { 0.0, 0.0, 0.0, 1.0 };
    GLfloat shininess = 65.0;

    glMaterialfv(GL_FRONT, GL_AMBIENT, ambient);
    glMaterialfv(GL_FRONT, GL_DIFFUSE, diffuse);
    glMaterialfv(GL_FRONT, GL_SPECULAR, specular);
    glMaterialf(GL_FRONT, GL_SHININESS, shininess);

    if (model_list)
        glDeleteLists(model_list, 1);

    glDisable(GL_TEXTURE_2D);
    /* generate a list */
    switch (material_mode)
    {
     case 0:
        if (facet_normal)
            model_list = glmList(model, GLM_FLAT);
        else
            model_list = glmList(model, GLM_SMOOTH);
     break;
     case 1:
        if (facet_normal)
            model_list = glmList(model, GLM_FLAT | GLM_COLOR);
        else
            model_list = glmList(model, GLM_SMOOTH | GLM_COLOR);
     break;
     case 2:
        if (facet_normal)
            model_list = glmList(model, GLM_FLAT | GLM_MATERIAL);
        else
            model_list = glmList(model, GLM_SMOOTH | GLM_MATERIAL);
     break;
     case 3:
        glEnable(GL_TEXTURE_2D);
        model_list = glmList(model, GLM_TEXTURE);
//        glDisable(GL_TEXTURE_2D);
     break;
    }
}

void init(void){
    gltbInit(GLUT_LEFT_BUTTON);

    /* read in the model */
    model = glmReadOBJ(model_file);
    scale = glmUnitize(model);
    glmFacetNormals(model);
    glmVertexNormals(model, smoothing_angle);

    if (model->nummaterials > 0)
        material_mode = 2;

    /* create new display lists */
    lists();

    glEnable(GL_LIGHTING);
    glEnable(GL_LIGHT0);
    glLightModeli(GL_LIGHT_MODEL_TWO_SIDE, GL_TRUE);
    glEnable(GL_DEPTH_TEST);
}

void reshape(int width, int height){
    gltbReshape(width, height);

    glViewport(0, 0, width, height);

    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    gluPerspective(60.0, (GLfloat)height / (GLfloat)width, 1.0, 128.0);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glTranslatef(0.0, 0.0, -3.0);
}

void display(void){
    static char s[256], t[32];
    static char* p;
    static int frames = 0;
    int i=0,j=0;

    glClearColor(1.0, 1.0, 1.0, 1.0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    glPushMatrix();

    glTranslatef(pan_x, pan_y, 0.0);

    gltbMatrix();

    if(mouseClicked){
        line();
    }

    glCallList(model_list);

    glPopMatrix();

    glutSwapBuffers();
    glEnable(GL_LIGHTING);
}

static GLint      mouse_state;
static GLint      mouse_button;

void mouse(int button, int state, int x, int y){
    GLdouble model_project[4*4];
    GLdouble proj[4*4];
    GLint view[4];

    /* fix for two-button mice -- left mouse + shift = middle mouse */
    if (button == GLUT_LEFT_BUTTON && glutGetModifiers() & GLUT_ACTIVE_SHIFT)
        button = GLUT_MIDDLE_BUTTON;

    gltbMouse(button, state, x, y);

    mouse_state = state;
    mouse_button = button;

    if (state == GLUT_DOWN && button == GLUT_MIDDLE_BUTTON) {
        glGetDoublev(GL_MODELVIEW_MATRIX, model_project);
        glGetDoublev(GL_PROJECTION_MATRIX, proj);
        glGetIntegerv(GL_VIEWPORT, view);

        gluProject((GLdouble)x, (GLdouble)y, 0.0,
            model_project, proj, view,
            &pan_x, &pan_y, &pan_z);
        gluUnProject((GLdouble)x, (GLdouble)y, pan_z,
            model_project, proj, view,
            &pan_x, &pan_y, &pan_z);
        pan_y = -pan_y;
    }

    if(button == GLUT_LEFT_BUTTON && state == GLUT_DOWN && glutGetModifiers() & GLUT_ACTIVE_CTRL) {
        convert2dto3D(x,y);
    }
    else if(button == GLUT_WHEEL_UP){ // Wheel up
        glmScale(model, 1.25);
        lists();
    }
    else if(button == GLUT_WHEEL_DOWN){ // Wheel down
        glmScale(model, 0.8);
        lists();
    }

    glutPostRedisplay();
}

void motion(int x, int y){
    GLdouble model[4*4];
    GLdouble proj[4*4];
    GLint view[4];

    gltbMotion(x, y);


    glutPostRedisplay();
}

int main(int argc, char** argv){
    int buffering = GLUT_DOUBLE;
    struct dirent* direntp;
    DIR* dirp;
    int models;

    glutInitWindowSize(512, 512);
    glutInit(&argc, argv);

    while (--argc) {
        if (strcmp(argv[argc], "-sb") == 0)
            buffering = GLUT_SINGLE;
        else
            model_file = argv[argc];
    }

    if (!model_file) {
//        model_file = "data/dolphins.obj";
        model_file = "data/boeing_2.obj";
    }

    glutInitDisplayMode(GLUT_RGB | GLUT_DEPTH | buffering);
    glutCreateWindow("Smooth");

    glutReshapeFunc(reshape);
    glutDisplayFunc(display);
    glutKeyboardFunc(keyboard);
    glutMouseFunc(mouse);
    glutMotionFunc(motion);

/* Image data packed tightly. */
    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);

    textures();

    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL);
//    glEnable(GL_TEXTURE_2D);

    models = glutCreateMenu(menu);
    dirp = opendir(DATA_DIR);
    if (!dirp) {
        fprintf(stderr, "%s: can't open data directory.\n", argv[0]);
    } else {
        while ((direntp = readdir(dirp)) != NULL) {
            if (strstr(direntp->d_name, ".obj")) {
                entries++;
                glutAddMenuEntry(direntp->d_name, -entries);
            }
        }
        closedir(dirp);
    }


    init();

    glutMainLoop();
    return 0;
}

Here trackball's functions:

     /*
     *  Simple trackball-like motion adapted (ripped off) from projtex.c
     *  (written by David Yu and David Blythe).  See the SIGGRAPH '96
     *  Advanced OpenGL course notes.
     */


    #include <math.h>
    #include <stdio.h>
    #include <assert.h>
    #include <GL/glut.h>
    #include "gltb.h"


    #define GLTB_TIME_EPSILON  10


    static GLuint    gltb_lasttime;
    static GLfloat   gltb_lastposition[3];

    static GLfloat   gltb_angle = 0.0;
    static GLfloat   gltb_axis[3];
    static GLfloat   gltb_transform[4][4];

    static GLuint    gltb_width;
    static GLuint    gltb_height;

    static GLint     gltb_button = -1;
    static GLboolean gltb_tracking = GL_FALSE;
    static GLboolean gltb_animate = GL_TRUE;


    static void
    _gltbPointToVector(int x, int y, int width, int height, float v[3])
    {
      float d, a;

      /* project x, y onto a hemi-sphere centered within width, height. */
      v[0] = (2.0 * x - width) / width;
      v[1] = (height - 2.0 * y) / height;
      d = sqrt(v[0] * v[0] + v[1] * v[1]);
      v[2] = cos((3.14159265 / 2.0) * ((d < 1.0) ? d : 1.0));
      a = 1.0 / sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
      v[0] *= a;
      v[1] *= a;
      v[2] *= a;
    }

    static void
    _gltbAnimate(void)
    {
      glutPostRedisplay();
    }

    void
    _gltbStartMotion(int x, int y, int button, int time)
    {
      assert(gltb_button != -1);

      gltb_tracking = GL_TRUE;
      gltb_lasttime = time;
      _gltbPointToVector(x, y, gltb_width, gltb_height, gltb_lastposition);
    }

    void
    _gltbStopMotion(int button, unsigned time)
    {
      assert(gltb_button != -1);

      gltb_tracking = GL_FALSE;

      if (time - gltb_lasttime < GLTB_TIME_EPSILON && gltb_animate) {
          glutIdleFunc(_gltbAnimate);
      } else {
        gltb_angle = 0;
        if (gltb_animate)
          glutIdleFunc(0);
      }
    }

    void
    gltbAnimate(GLboolean animate)
    {
      gltb_animate = animate;
    }

    void
    gltbInit(GLuint button)
    {
      gltb_button = button;
      gltb_angle = 0.0;

      /* put the identity in the trackball transform */
      glPushMatrix();
      glLoadIdentity();
      glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)gltb_transform);
      glPopMatrix();
    }

    void
    gltbMatrix(void)
    {
      assert(gltb_button != -1);

      glPushMatrix();
      glLoadIdentity();
      glRotatef(gltb_angle, gltb_axis[0], gltb_axis[1], gltb_axis[2]);
      glMultMatrixf((GLfloat*)gltb_transform);
      glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)gltb_transform);
      glPopMatrix();

      glMultMatrixf((GLfloat*)gltb_transform);
    }

    void
    gltbReshape(int width, int height)
    {
      assert(gltb_button != -1);

      gltb_width  = width;
      gltb_height = height;
    }

    void
    gltbMouse(int button, int state, int x, int y)
    {
      assert(gltb_button != -1);

      if (state == GLUT_DOWN && button == gltb_button)
        _gltbStartMotion(x, y, button, glutGet(GLUT_ELAPSED_TIME));
      else if (state == GLUT_UP && button == gltb_button)
        _gltbStopMotion(button, glutGet(GLUT_ELAPSED_TIME));
    }

    void
    gltbMotion(int x, int y)
    {
      GLfloat current_position[3], dx, dy, dz;

      assert(gltb_button != -1);

      if (gltb_tracking == GL_FALSE)
        return;

      _gltbPointToVector(x, y, gltb_width, gltb_height, current_position);

      /* calculate the angle to rotate by (directly proportional to the
         length of the mouse movement) */
      dx = current_position[0] - gltb_lastposition[0];
      dy = current_position[1] - gltb_lastposition[1];
      dz = current_position[2] - gltb_lastposition[2];
      gltb_angle = 90.0 * sqrt(dx * dx + dy * dy + dz * dz);

      /* calculate the axis of rotation (cross product) */
      gltb_axis[0] = gltb_lastposition[1] * current_position[2] - 
                   gltb_lastposition[2] * current_position[1];
      gltb_axis[1] = gltb_lastposition[2] * current_position[0] - 
                   gltb_lastposition[0] * current_position[2];
      gltb_axis[2] = gltb_lastposition[0] * current_position[1] - 
                   gltb_lastposition[1] * current_position[0];

      /* XXX - constrain motion */
      gltb_axis[2] = 0;

      /* reset for next time */
      gltb_lasttime = glutGet(GLUT_ELAPSED_TIME);
      gltb_lastposition[0] = current_position[0];
      gltb_lastposition[1] = current_position[1];
      gltb_lastposition[2] = current_position[2];

      /* remember to draw new position */
      glutPostRedisplay();
    }

If I cast a ray (CTRL + Left_click) at the beginning It works with correct 3D far_point and near_point If I rotate the object (use Trackball), the ray doesn't use real eye/far points and I can't figure out why. I've tried with GluLookAt instead of glRotatef, but I can't figure out how to move camera with mouse like this trackball does.

Answer 1

Solved: The problem was the pushMatrix()/popMatrix() at beginning/end in display().