Getting current frame from android vlc media player

Question

I'm using VLC-Android in order to play H264 RTSP live stream in my android application, the following code successfully renders the stream video onto the surface view:

MediaPlayer mMediaPlayer = new MediaPlayer(VLCInstance.get());
SurfaceView mSurfaceView = (SurfaceView) findViewById(R.id.player);

final IVLCVout vlcVout = Constants.mMediaPlayer.getVLCVout();
vlcVout.detachViews();
vlcVout.setVideoView(mSurfaceView);
vlcVout.setWindowSize(mSurfaceDims.getWidth(), mSurfaceDims.getHeight());
vlcVout.attachViews();
mSurfaceView.setKeepScreenOn(true);

Media media = new Media(VLCInstance.get(), Uri.parse(path));
mMediaPlayer.setMedia(media);
mMediaPlayer.play();

VLCInstance.java:

package bi.anpr.vlc;

import android.content.Context;
import android.util.Log;

import org.videolan.libvlc.LibVLC;
import org.videolan.libvlc.util.VLCUtil;

public class VLCInstance {
    public final static String TAG = "VLC/Util/VLCInstance";

    private static LibVLC sLibVLC = null;

    /** A set of utility functions for the VLC application */
    public synchronized static LibVLC get() throws IllegalStateException {
        if (sLibVLC == null) {
            final Context context = VLCApplication.getAppContext();
            if(!VLCUtil.hasCompatibleCPU(context)) {
                Log.e(TAG, VLCUtil.getErrorMsg());
                throw new IllegalStateException("LibVLC initialisation failed: " + VLCUtil.getErrorMsg());
            }

            try{
                sLibVLC = new LibVLC(context);
            }catch (Throwable e){
                e.printStackTrace();
            }

        }
        return sLibVLC;
    }

    public static synchronized void restart(Context context) throws IllegalStateException {
        if (sLibVLC != null) {
            sLibVLC.release();
            sLibVLC = new LibVLC(context);
        }
    }
}

The problem is that the org.videolan.libvlc.MediaPlayer renders the video on the SurfaceView after calling play() without any event that helps you to grab the current frame on the view, where i need to get the current frame each interval of time to perform some image processing on it.

So is it possible using vlc-android-sdk to grab current frame, into a buffered image or an OpenCV Mat, while playing live streams or local video resources?

Note that it is possible with java (desktop vlcj) through the DirectMediaPlayer class.

Thanks in advance!!

Answer 1

I'd found a very simple an very fast solution to do this. Just use a TextureView instead of a SurfaceView, and then retrieve the image anytime while playing using the getBitmap() function as shown below.

public class MainActivity extends AppCompatActivity implements TextureView.SurfaceTextureListener,
        org.videolan.libvlc.media.MediaPlayer.OnBufferingUpdateListener,
        org.videolan.libvlc.media.MediaPlayer.OnCompletionListener,
        org.videolan.libvlc.media.MediaPlayer.OnPreparedListener,
        org.videolan.libvlc.media.MediaPlayer.OnVideoSizeChangedListener {

    private AppCompatActivity me = this;    
    private MediaPlayer mMediaPlayer;    
    private TextureView mTextureViewmTextureView;
    private String mUrl = "/storage/emulated/0/videos/test.mp4";
    private static final String TAG = "MainActivity";

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.content_main);

        mMediaPlayer = new MediaPlayer(VLCInstance.get());
        mTextureViewmTextureView = (TextureView) findViewById(R.id.player);
        mTextureView.setSurfaceTextureListener(this);
    }

    private void attachViewSurface() {
            final IVLCVout vlcVout = mMediaPlayer.getVLCVout();
            mMediaPlayer.setScale(0);
            vlcVout.detachViews();
            vlcVout.setVideoView(mTextureView);
            vlcVout.setWindowSize(mTextureView.getWidth(), mTextureView.getHeight());
            vlcVout.attachViews();
            mTextureView.setKeepScreenOn(true);
    }


    private void play(String path) {
       try {
            Media media;
            if (new File(path).exists()) {
                media = new Media(VLCInstance.get(), path);
            } else {
                media = new Media(VLCInstance.get(), Uri.parse(path));
            }

            mMediaPlayer.setMedia(media);
            mMediaPlayer.play();
        } catch (Exception e) {
            Log.e(TAG, e.getMessage());
        }
   }

   @Override
   public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
       attachViewSurface();

       if (mMediaPlayer.hasMedia())
           mMediaPlayer.play();
       else
           play(mUrl);
   }

   public Bitmap getImage() {
       return mTextureView.getBitmap();
   }

}

VLCInstance.java:

import android.content.Context;
import android.util.Log;

import org.videolan.libvlc.LibVLC;
import org.videolan.libvlc.util.VLCUtil;

public class VLCInstance {
    public final static String TAG = "VLC/Util/VLCInstance";

    private static LibVLC sLibVLC = null;

    /** A set of utility functions for the VLC application */
    public synchronized static LibVLC get() throws IllegalStateException {
        if (sLibVLC == null) {
            final Context context = VLCApplication.getAppContext();
            if(!VLCUtil.hasCompatibleCPU(context)) {
                Log.e(TAG, VLCUtil.getErrorMsg());
                throw new IllegalStateException("LibVLC initialisation failed: " + VLCUtil.getErrorMsg());
            }

            try{
                sLibVLC = new LibVLC(context);
            }catch (Throwable e){
                e.printStackTrace();
            }

        }
        return sLibVLC;
    }

    public static synchronized void restart(Context context) throws IllegalStateException {
        if (sLibVLC != null) {
            sLibVLC.release();
            sLibVLC = new LibVLC(context);
        }
    }
}