OpenGl: Problem with Arcball Camera Rotation

Question

i need to implement arcball camera. I got something similar, but it works very crookedly (the angle changes sharply, when turning to the right / left, the camera raises up / down strongly).

Here is my source code, can you tell me where I went wrong:

bool get_arcball_vec(double x, double y, glm::vec3& a) 
{

  glm::vec3 vec = glm::vec3((2.0 * x) / window.getWidth() - 1.0, 1.0 - (2.0 * y) / window.getHeight(), 0.0);


 if (glm::length(vec) >= 1.0)
 {
   vec = glm::normalize(vec);
 }
 else 
 {
   vec.z = sqrt(1.0 - pow(vec.x, 2.0) - pow(vec.y, 2.0));
 } 
 a = vec;

return true;
}

...
void onMouseMove(double x, double y) {
  if (rightMouseButtonPressed) {
    glm::vec3 a,b;
    cur_mx = x;
    cur_my = y;
    if (cur_mx != last_mx || cur_my != last_my)
      if (get_arcball_vec(last_mx, last_my, a) && get_arcball_vec(cur_mx, cur_my, b))
        viewport.getCamera().orbit(a,b);
  
    last_mx = cur_mx;
    last_my = cur_my;

      ...

void Camera::orbit(glm::vec3 a, glm::vec3 b)
{
     forward = calcForward();
     right = calcRight();


    double alpha = acos(glm::min(1.0f, glm::dot(b, a)));


     glm::vec3 axis = glm::cross(a, b);


     glm::mat4 rotationComponent = glm::mat4(1.0f);

    rotationComponent[0] = glm::vec4(right, 0.0f);
    rotationComponent[1] = glm::vec4(up, 0.0f);
    rotationComponent[2] = glm::vec4(forward, 0.0f);


    glm::mat4 toWorldCameraSpace = glm::transpose(rotationComponent);


    axis = toWorldCameraSpace * glm::vec4(axis, 1.0);

    glm::mat4 orbitMatrix = glm::rotate(glm::mat4(1.0f), (float)alpha, axis);


    eye = glm::vec4(target, 1.0) + orbitMatrix * glm::vec4(eye - target, 1.0f);

    up = orbitMatrix * glm::vec4(up, 1.0f);
   }

Answer 1

I use this code to map 2D mouse position to the sphere:

Vector3 GetArcBallVector(const Vector2f & mousePos) {

    float radiusSquared = 1.0; //squared radius of the sphere

    //compute mouse position from the centre of screen to interval [-half, +half]
    Vector3 pt = Vector3(
       mousePos.x - halfScreenW,
       halfScreenH - mousePos.y,
       0.0f
    );

    //if length squared is smaller than sphere diameter
    //point is inside
    float lengthSqr = pt.x * pt.x + pt.y * pt.y;
            
    if (lengthSqr < radiusSquared){
        //inside        
       pt.z = std::sqrtf(radiusSquared - lengthSqr);
    }
    else {              
       pt.z = 0.0f;     
    }

    pt.z *= -1;
    return pt;
}

To calculate rotation, I use the last (startPt) and current (endPt) mapped position and do:

Quaternion actRot = Quaternion::Identity();
Vector3 axis = Vector3::Cross(endPt, startPt);
    
if (axis.LengthSquared() > MathUtils::EPSILON) {        
    float angleCos = Vector3::Dot(endPt, startPt);
    actRot = Quaternion(axis.x, axis.y, axis.z, angleCos);      
}

I prefer to use Quaternions over matrices since they are easy to multiply (for acumulated rotation) and interpolate (for some smooting).