Depth representation in disparity map rendered using SGBM is wildly inaccurate

Question

I am creating a disparity map using SGBM in OpenCV along with WLSFilter. Although the objects are distinct in the output, the depth representation is way off and random - it looks like a greycale painting of the original image. This makes me wonder if something else is wrong with my code. Would appreciate any help I can get to figure this out.

Sample image 1 Ground truth My output 1 with BM Block size = 5; NumDisp = 16; Greyscale = true; My output 1 with SGBM Block size = 5; NumDisp = 10;

Sample image 2 My output 2 Block size = 5; NumDisp = 50;

Code:

void StereoMatching::computeDisparity(cv::Mat &left, cv::Mat &right, cv::Mat &disp) {
  bool gray_scale = false;
  std::string algorithm = "StereoSGBM";
  int blur_kernel = 3;

  int block_size = 5, min_disp = 0, max_disp = 50, max_diff = 0, pre_fc = 0, unique_ratio = 0,
      speckle_size = 50, speckle_range = 10;
                            
  bool use_filter = true;                     
  int p1 = 8 * 3 * block_size * block_size;    
  int p2 = 32 * 3 * block_size * block_size;   

  double wsl_lambda = 6000.0, wsl_sigma = 1.0;

  if (gray_scale) {
    cv::cvtColor(left, left, cv::COLOR_BGR2GRAY);
    cv::cvtColor(right, right, cv::COLOR_BGR2GRAY);
  }

  cv::blur(left, left, cv::Size(blur_kernel, blur_kernel));
  cv::blur(right, right, cv::Size(blur_kernel, blur_kernel));
  cv::Ptr<cv::ximgproc::DisparityWLSFilter> wls_filter_;
  cv::Ptr<cv::StereoMatcher> stereo_matcher_;
  cv::Ptr<cv::StereoMatcher> stereo_matcher_r_;

  if (algorithm == "StereoBM") {
    stereo_matcher_ = cv::StereoBM::create(16, 5);
  }

  else if (algorithm == "StereoSGBM") {
    stereo_matcher_ =
        cv::StereoSGBM::create(min_disp, max_disp, block_size, p1, p2, max_diff, pre_fc,
                               unique_ratio, speckle_size, speckle_range, cv::StereoSGBM::MODE_HH);
  }

  if (use_filter) {
    wls_filter_ = cv::ximgproc::createDisparityWLSFilter(stereo_matcher_);
    stereo_matcher_r_ = cv::ximgproc::createRightMatcher(stereo_matcher_);
    wls_filter_->setLambda(wsl_lambda);
    wls_filter_->setSigmaColor(wsl_sigma);

    cv::Mat dispL, dispR;
    stereo_matcher_->compute(left, right, dispL);
    stereo_matcher_r_->compute(right, left, dispR);
    wls_filter_->filter(dispL, left, disp, dispR);
  }

  else {
    stereo_matcher_->compute(left, right, disp);
  }