The top left is the left camera, the top right is the right camera, the bottom left is the left camera converted to gray, and the bottom right is the disparity view.
Android Kotlin Activity Code
class ObjectDistanceActivity : BaseActivity(), CameraDialogParent {
private lateinit var viewBinding: ActivityObjectDistanceBinding
// for accessing USB and USB camera
private lateinit var usbMonitor: USBMonitor
private lateinit var handlerL: UVCCameraHandler
private lateinit var handlerR: UVCCameraHandler
private lateinit var cameraViewLeft: CameraViewInterface
private lateinit var cameraViewRight: CameraViewInterface
private lateinit var previewLeft: Surface
private lateinit var previewRight: Surface
private lateinit var distanceAnalyzer: DistanceAnalyzer
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
viewBinding = ActivityObjectDistanceBinding.inflate(layoutInflater)
setContentView(viewBinding.root)
if (!OpenCVLoader.initDebug()) {
Log.d(AppUtil.DEBUG_TAG, "OpenCV Error")
} else {
Log.d(AppUtil.DEBUG_TAG, "OpenCV Success")
}
distanceAnalyzer = DistanceAnalyzer(this)
distanceAnalyzer.resultListener = object : DistanceAnalyzer.OnResultListener {
override fun onResult(leftBitmap: Bitmap, rightBitmap: Bitmap) {
runOnUiThread {
viewBinding.disparityMap.setImageBitmap(leftBitmap)
viewBinding.objectDetect.setImageBitmap(rightBitmap)
}
}
}
cameraViewLeft = viewBinding.cameraViewLeft
cameraViewLeft.aspectRatio = (AppUtil.DEFAULT_WIDTH / AppUtil.DEFAULT_HEIGHT.toFloat()).toDouble()
(cameraViewLeft as UVCCameraTextureView).setOnClickListener(mOnClickListener)
handlerL = UVCCameraHandler.createHandler(this, cameraViewLeft, 2,
AppUtil.DEFAULT_WIDTH, AppUtil.DEFAULT_HEIGHT, UVCCamera.PIXEL_FORMAT_RGB565)
// handlerL = UVCCameraHandler.createHandler(this, cameraViewLeft, UVCCamera.DEFAULT_PREVIEW_WIDTH, UVCCamera.DEFAULT_PREVIEW_HEIGHT, BANDWIDTH_FACTORS[0])
cameraViewRight = viewBinding.cameraViewRight
cameraViewRight.aspectRatio = (AppUtil.DEFAULT_WIDTH / AppUtil.DEFAULT_HEIGHT.toFloat()).toDouble()
(cameraViewRight as UVCCameraTextureView).setOnClickListener(mOnClickListener)
// handlerR = UVCCameraHandler.createHandler(this, cameraViewRight, UVCCamera.DEFAULT_PREVIEW_WIDTH, UVCCamera.DEFAULT_PREVIEW_HEIGHT, BANDWIDTH_FACTORS[1])
handlerR = UVCCameraHandler.createHandler(this, cameraViewRight, 2,
AppUtil.DEFAULT_WIDTH, AppUtil.DEFAULT_HEIGHT, UVCCamera.PIXEL_FORMAT_RGB565)
usbMonitor = USBMonitor(this, mOnDeviceConnectListener)
Log.d(AppUtil.DEBUG_TAG, "oncreate")
}
override fun onStart() {
super.onStart()
usbMonitor.register()
cameraViewRight.onResume()
cameraViewLeft.onResume()
}
override fun onStop() {
handlerR.close()
cameraViewRight.onPause()
handlerL.close()
cameraViewLeft.onPause()
usbMonitor.unregister()
super.onStop()
}
override fun onDestroy() {
usbMonitor.destroy()
distanceAnalyzer.flag = false
super.onDestroy()
}
private val mOnClickListener = OnClickListener { view ->
when (view.id) {
R.id.camera_view_left -> if (!handlerL.isOpened) {
CameraDialog.showDialog(this@ObjectDistanceActivity)
} else {
handlerL.close()
setCameraButton()
}
R.id.camera_view_right -> if (!handlerR.isOpened) {
CameraDialog.showDialog(this@ObjectDistanceActivity)
} else {
handlerR.close()
setCameraButton()
}
}
}
private val mOnDeviceConnectListener: OnDeviceConnectListener = object : OnDeviceConnectListener {
override fun onAttach(device: UsbDevice) {
if (DEBUG) Log.v(TAG, "onAttach:$device")
Toast.makeText(this@ObjectDistanceActivity, "USB_DEVICE_ATTACHED", Toast.LENGTH_SHORT).show()
}
override fun onConnect(device: UsbDevice, ctrlBlock: USBMonitor.UsbControlBlock, createNew: Boolean) {
if (DEBUG) Log.v(TAG, "onConnect:$device")
if (!handlerL.isOpened) {
handlerL.open(ctrlBlock)
val st = cameraViewLeft.surfaceTexture
handlerL.setPreviewCallback(distanceAnalyzer.iFrameLeftCallback)
handlerL.startPreview(Surface(st))
} else if (!handlerR.isOpened) {
handlerR.open(ctrlBlock)
val st = cameraViewRight.surfaceTexture
handlerR.setPreviewCallback(distanceAnalyzer.iFrameRightCallback)
handlerR.startPreview(Surface(st))
}
}
override fun onDisconnect(device: UsbDevice, ctrlBlock: USBMonitor.UsbControlBlock) {
if (DEBUG) Log.v(TAG, "onDisconnect:$device")
if (!handlerL.isEqual(device)) {
queueEvent({
handlerL.close()
previewLeft.release()
setCameraButton()
}, 0)
} else if (!handlerR.isEqual(device)) {
queueEvent({
handlerR.close()
previewRight.release()
setCameraButton()
}, 0)
}
}
override fun onDettach(device: UsbDevice) {
if (DEBUG) Log.v(TAG, "onDettach:$device")
Toast.makeText(this@ObjectDistanceActivity, "USB_DEVICE_DETACHED", Toast.LENGTH_SHORT).show()
}
override fun onCancel(device: UsbDevice) {
if (DEBUG) Log.v(TAG, "onCancel:")
}
}
/**
* to access from CameraDialog
* @return
*/
override fun getUSBMonitor(): USBMonitor {
return usbMonitor
}
override fun onDialogResult(canceled: Boolean) {
if (canceled) {
runOnUiThread(Runnable { setCameraButton() }, 0)
}
}
private fun setCameraButton() {
}
companion object {
private const val DEBUG = false // FIXME set false when production
private const val TAG = "MainActivity"
private val BANDWIDTH_FACTORS = floatArrayOf(0.5f, 0.5f)
}}
`
Android Kotlin Code
\`
class DistanceAnalyzer(val context: Context) {
private val lock = Any()
private var initFinished = false
private var lbpCascadeClassifier: CascadeClassifier? = null
private var leftByteBuffer: ByteBuffer? = null
private var rightByteBuffer: ByteBuffer? = null
var flag = true
val iFrameLeftCallback = IFrameCallback {
synchronized(lock) {
leftByteBuffer = it.duplicate()
if (leftByteBuffer != null && rightByteBuffer != null && initFinished) {
analyze(leftByteBuffer!!.duplicate(), rightByteBuffer!!.duplicate())
}
}
}
val iFrameRightCallback = IFrameCallback {
synchronized(lock) {
rightByteBuffer = it.duplicate()
}
}
interface OnResultListener {
fun onResult(leftBitmap: Bitmap, rightBitmap: Bitmap)
}
interface OnCalibrateFinished {
fun onSuccess()
}
private val onHaarCascadeInit = object : OnCalibrateFinished {
override fun onSuccess() {
initFinished = true
}
}
var resultListener: OnResultListener? = null
init {
val inputStream = context.resources.openRawResource(org.opencv.R.raw.lbpcascade_frontalface)
val file = File(context.getDir(
"cascade", BaseActivity.MODE_PRIVATE
),
"lbpcascade_frontalface.xml")
val fileOutputStream = FileOutputStream(file)
// asd
val data = ByteArray(4096)
var readBytes: Int
while (inputStream.read(data).also { readBytes = it } != -1) {
fileOutputStream.write(data, 0, readBytes)
}
lbpCascadeClassifier = CascadeClassifier(file.absolutePath)
inputStream.close()
fileOutputStream.close()
file.delete()
onHaarCascadeInit.onSuccess()
}
private fun analyze(leftBuffer: ByteBuffer, rightBuffer: ByteBuffer) {
synchronized(lock) {
val srcLeftBitmap = Bitmap.createBitmap(
AppUtil.DEFAULT_WIDTH,
AppUtil.DEFAULT_HEIGHT,
Bitmap.Config.RGB_565
)
srcLeftBitmap.copyPixelsFromBuffer(leftBuffer)
val srcRightBitmap = Bitmap.createBitmap(
AppUtil.DEFAULT_WIDTH,
AppUtil.DEFAULT_HEIGHT,
Bitmap.Config.RGB_565
)
srcRightBitmap.copyPixelsFromBuffer(rightBuffer)
val rgbLeftMat = Mat(AppUtil.DEFAULT_WIDTH, AppUtil.DEFAULT_HEIGHT, CvType.CV_8UC3)
val rgbRightMat = Mat(AppUtil.DEFAULT_WIDTH, AppUtil.DEFAULT_HEIGHT, CvType.CV_8UC3)
Utils.bitmapToMat(srcLeftBitmap, rgbLeftMat)
Utils.bitmapToMat(srcRightBitmap, rgbRightMat)
// gray convert
val grayLeftMat = Mat()
Imgproc.cvtColor(rgbLeftMat, grayLeftMat, Imgproc.COLOR_RGB2GRAY)
val grayRightMat = Mat()
Imgproc.cvtColor(rgbRightMat, grayRightMat, Imgproc.COLOR_RGB2GRAY)
val grayLeftBitmap = Bitmap.createBitmap(AppUtil.DEFAULT_WIDTH, AppUtil.DEFAULT_HEIGHT, Bitmap.Config.RGB_565)
Utils.matToBitmap(grayLeftMat, grayLeftBitmap)
// create stereo SGBM
val minDisp = 2
val numDisp = 130 - minDisp
val windowSize = 3
val stereo = StereoSGBM.create(minDisp,
numDisp, windowSize, 10, 100,
32, 5,
8*3*windowSize.toDouble().pow(2).toInt(),
32*3*windowSize.toDouble().pow(2).toInt()
)
// cal disparity map
val disparityMat = Mat()
stereo.compute(grayRightMat, grayLeftMat, disparityMat)
Log.d(AppUtil.DEBUG_TAG, disparityMat.toString())
val normalizedMat = Mat()
Core.normalize(disparityMat, normalizedMat, 0.0, 256.0, Core.NORM_MINMAX)
normalizedMat.convertTo(normalizedMat, CvType.CV_8U)
// convert dispairty bitmap
val disparityBitmap = Bitmap.createBitmap(normalizedMat.width(), normalizedMat.height(), Bitmap.Config.RGB_565)
Utils.matToBitmap(normalizedMat, disparityBitmap)
// output
resultListener?.onResult(grayLeftBitmap, disparityBitmap)
}
}
private fun faceDistance(x: Int, y: Int, dispNorm: Mat): Double {
var average = 0.0
for (u in -1..1) {
for (v in -1..1) {
val value = dispNorm.get(y + u, x + v)
average += value.sum()
Log.d(AppUtil.DEBUG_TAG, value.toString())
Log.d(AppUtil.DEBUG_TAG, "$average average")
}
}
average /= 9.0
return -593.97 * average.pow(3) + 1506.8 * average.pow(2) - 1373.1 * average + 522.06
}}
I am trying to create a stereo vision application using the libusbcamera library and OpenCV 4.8 library. I have been working based on the GitHub repository at https://github.com/LearnTechWithUs/Stereo-Vision. I have successfully executed the process in Python, excluding the calibration part, and confirmed its functionality. However, when I attempted to convert it to Kotlin for Android implementation, the generated disparity map only shows a black screen. Could I be missing something?
I tried the solutions from both my 'Disparity map' result is black? and Depth map - stereo image in Android with OpenCV, but there were no changes.