Unity Agora SDK - AR Camera Issue

Question

Hi I was trying to follow this guide: https://www.agora.io/en/blog/video-chat-with-unity3d-ar-foundation-pt3-remote-assistance-app/

Basically, I want to create a remote assistance app to connect the technician to the viewer in Augmented Reality. The problem is, even after wrestling with the code for days, I'm unable to send my AR camera video feed from the broadcaster to other people. Also the Agora SDK has changed so much so I'm not sure if I'm implementing code correctly. Need help with this please.

Agora Video SDK Version: v4.2.6 Unity Version: 2019.4.40.f1

Here's my TechnicianManager.cs

using System.Collections;
using System.Collections.Generic;
using Agora_RTC_Plugin.API_Example.Examples.Advanced.WriteBackVideoRawData;
using Agora.Rtc;
using UnityEngine;
using UnityEngine.Android;
using UnityEngine.SceneManagement;
using UnityEngine.UI;
using UnityEngine.XR.ARFoundation;

public class TechnicianManager : MonoBehaviour
{
    private IRtcEngine mRtcEngine;
    [SerializeField] private ARCameraManager cameraManager;

    [SerializeField] public string appId;
    [SerializeField] public string channelName;
    
    private Texture2D BufferTexture;
    private static TextureFormat ConvertFormat = TextureFormat.BGRA32;
    private static VIDEO_PIXEL_FORMAT PixelFormat = VIDEO_PIXEL_FORMAT.VIDEO_PIXEL_BGRA;
    
    private int i = 0; // monotonic timestamp counter

    private void SetupAgoraEngine()
    {
        mRtcEngine = Agora.Rtc.RtcEngine.CreateAgoraRtcEngine();
        
        UserEventHandler handler = new UserEventHandler(this);
        
        RtcEngineContext context = new RtcEngineContext(appId, 0,
            CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_LIVE_BROADCASTING,
            AUDIO_SCENARIO_TYPE.AUDIO_SCENARIO_DEFAULT, AREA_CODE.AREA_CODE_GLOB);
        
        mRtcEngine.Initialize(context);
        mRtcEngine.InitEventHandler(handler);
        
        mRtcEngine.SetVideoEncoderConfiguration(new VideoEncoderConfiguration
        {
            dimensions = new VideoDimensions{width = 360, height = 640},
            frameRate = 24,
            bitrate = 800,
            orientationMode = ORIENTATION_MODE.ORIENTATION_MODE_FIXED_PORTRAIT
        });
        
        // allow camera output callback
        //mRtcEngine.EnableVideoObserver();
        //mRtcEngine.EnableLocalVideo(false);

        //mRtcEngine.RegisterVideoFrameObserver(new VideoFrameObserver(this), VIDEO_OBSERVER_FRAME_TYPE.FRAME_TYPE_NV21,
        //    VIDEO_OBSERVER_POSITION.POSITION_POST_CAPTURER, OBSERVER_MODE.RAW_DATA);
        
        //  mRtcEngine.SetVideoQualityParameters(true);

        SenderOptions senderOptions = new SenderOptions
        {
            targetBitrate = 800
        };
        //senderOptions.
        
        mRtcEngine.SetExternalVideoSource(true, false, EXTERNAL_VIDEO_SOURCE_TYPE.VIDEO_FRAME, senderOptions);
        
        // enable video
        mRtcEngine.EnableAudio();
        mRtcEngine.EnableVideo();
        
        //mRtcEngine.SetChannelProfile(CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_COMMUNICATION);
        mRtcEngine.SetClientRole(CLIENT_ROLE_TYPE.CLIENT_ROLE_BROADCASTER);
        
        mRtcEngine.JoinChannel("", channelName);

        // Optional: if a data stream is required, here is a good place to create it
        int streamID = 0;
        DataStreamConfig dataStreamConfig = new DataStreamConfig();
        dataStreamConfig.ordered = true;
        mRtcEngine.CreateDataStream(ref streamID, dataStreamConfig);
        Debug.Log("ARHelper: initializeEngine done, data stream id = " + streamID);
    }
    
    // Start is called before the first frame update
    void Start()
    {
        GameObject go = GameObject.Find("myImage");
        if (go == null)
        {
            return;
        }

        VideoSurface videoSurface = go.AddComponent<VideoSurface>();
        videoSurface.enabled = false;


        go = GameObject.Find("ButtonExit");
        if (go != null)
        {
            Button button = go.GetComponent<Button>();
            if (button != null)
            {
                button.onClick.AddListener(OnLeaveButtonClicked);
            }
        }
        SetupToggleMic();
        
        go = GameObject.Find("ButtonColor");
        if (go != null)
        {
            // the button is only available for AudienceVC
            go.SetActive(false);
        }

        go = GameObject.Find("AR Camera");
        if (go != null)
        {
            cameraManager = go.GetComponent<ARCameraManager>();
        }


        go = GameObject.Find("sphere");
        if (go != null)
        {
            var sphere = go;
            // hide this before AR Camera start capturing
            sphere.SetActive(false);
             
            this.StartCoroutine(DelayAction(.5f,
                () =>
                {
                    sphere.SetActive(true);
                }));
        }
        
        SetupAgoraEngine();
    }

    void EnableSharing()
    {
        cameraManager.frameReceived += OnCameraFrameReceived;
        if (Camera.main != null)
        {
            RenderTexture renderTexture = Camera.main.targetTexture;
            if (renderTexture != null)
            {
                BufferTexture = new Texture2D(renderTexture.width, renderTexture.height, ConvertFormat, false);

                // Editor only, where onFrameReceived won't invoke
                if (Application.platform == RuntimePlatform.OSXEditor || Application.platform == RuntimePlatform.WindowsEditor)
                {
                    Debug.LogWarning(">>> Testing in Editor, start coroutine to capture Render data");
                    StartCoroutine(CoShareRenderData());
                }
            }
            else
            {
                Debug.LogError("ARHelper: renderTexture is NULL");
            }
        }
        else
        {
            Debug.LogError("ARHelper: Camera.main is NULL");
        }
    }
    
    /// <summary>
    ///    For use in Editor testing only.
    /// </summary>
    /// <returns></returns>
    IEnumerator CoShareRenderData()
    {
        yield return new WaitForEndOfFrame();
        OnCameraFrameReceived(default);
        yield return null;
    }
    
    /// <summary>
    ///   Delegate callback handles every frame generated by the AR Camera.
    /// </summary>
    /// <param name="eventArgs"></param>
    private void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
    {
        Debug.Log("ARHelper: OnCameraFrameReceived");
        if (BufferTexture == null) // offlined
        {
            Debug.LogError("ARHelper: BufferTexture is NULL");
            return;
        }
        Camera targetCamera = Camera.main; // AR Camera
        RenderTexture.active = targetCamera.targetTexture; // the targetTexture holds render texture
        Rect rect = new Rect(0, 0, targetCamera.targetTexture.width, targetCamera.targetTexture.height);
        BufferTexture.ReadPixels(rect, 0, 0);
        BufferTexture.Apply();

        byte[] bytes = BufferTexture.GetRawTextureData();
        
        Debug.Log("Bytes obtained from buffer with byteLength: " + bytes.Length);
        
        // sends the Raw data contained in bytes
        StartCoroutine(PushFrame(bytes, (int)rect.width, (int)rect.height,
            () =>
            {
                bytes = null;
            }));
        RenderTexture.active = null;
    }
    
    /// <summary>
    /// Push frame to the remote client.  This is the same code that does ScreenSharing.
    /// </summary>
    /// <param name="bytes">raw video image data</param>
    /// <param name="width"></param>
    /// <param name="height"></param>
    /// <param name="onFinish">callback upon finish of the function</param>
    /// <returns></returns>
    IEnumerator PushFrame(byte[] bytes, int width, int height, System.Action onFinish)
    {
        if (bytes == null || bytes.Length == 0)
        {
            Debug.LogError("ARHelper: Zero bytes found!!!!");
            yield break;
        }
        
        //if the engine is present
        if (mRtcEngine != null)
        {
            //Create a new external video frame
            ExternalVideoFrame externalVideoFrame = new ExternalVideoFrame();
            //Set the buffer type of the video frame
            externalVideoFrame.type = VIDEO_BUFFER_TYPE.VIDEO_BUFFER_RAW_DATA;
            // Set the video pixel format
            externalVideoFrame.format = PixelFormat; // VIDEO_PIXEL_BGRA for now
            //apply raw data you are pulling from the rectangle you created earlier to the video frame
            externalVideoFrame.buffer = bytes;
            //Set the width of the video frame (in pixels)
            externalVideoFrame.stride = width;
            //Set the height of the video frame
            externalVideoFrame.height = height;
            //Remove pixels from the sides of the frame
            externalVideoFrame.cropLeft = 10;
            externalVideoFrame.cropTop = 10;
            externalVideoFrame.cropRight = 10;
            externalVideoFrame.cropBottom = 10;
            //Rotate the video frame (0, 90, 180, or 270)
            externalVideoFrame.rotation = 180;
            // increment i with the video timestamp
            externalVideoFrame.timestamp = i++;
            //Push the external video frame with the frame we just created
            // int a = 
            int successCode = mRtcEngine.PushVideoFrame(externalVideoFrame);
            Debug.Log("ARHelper: Pushing Code: " + successCode);
            // Debug.Log(" pushVideoFrame(" + i + ") size:" + bytes.Length + " => " + a);
        }
        else
            Debug.LogError("ARHelper: RTC ENGINE NULL!");
        
        yield return null;
        onFinish();
    }
    
    private void SetupToggleMic()
    {
        GameObject go = GameObject.Find("ToggleButton");
        if (go != null)
        {
            ToggleButton toggle = go.GetComponent<ToggleButton>();
            if (toggle != null)
            {
                toggle.button1.onClick.AddListener(() =>
                {
                    toggle.Tap();
                    mRtcEngine.EnableLocalAudio(false);
                    mRtcEngine.MuteLocalAudioStream(true);
                });
                toggle.button2.onClick.AddListener(() =>
                {
                    toggle.Tap();
                    mRtcEngine.EnableLocalAudio(true);
                    mRtcEngine.MuteLocalAudioStream(false);
                });
            }
        }
    }
    private void OnLeaveButtonClicked()
    {
        // leave channel
        mRtcEngine.LeaveChannel();
        // deregister video frame observers in native-c code
        //mRtcEngine.DisableVideo();
        //mRtcEngine.DisableAudio();
        
        // delete engine
        
        mRtcEngine.Dispose();  // Place this call in ApplicationQuit
        mRtcEngine = null;
        
        SceneManager.LoadScene("MainScene", LoadSceneMode.Single);
    }
    
    IEnumerator DelayAction(float delay, System.Action doAction)
    {
        yield return new WaitForSeconds(delay);
        doAction();
    }
    
    internal class UserEventHandler : IRtcEngineEventHandler
    {
        private readonly TechnicianManager _technicianManager;

        internal UserEventHandler(TechnicianManager technicianManager)
        {
            _technicianManager = technicianManager;
        }
        
        public override void OnError(int err, string msg)
        {
            Debug.LogError(string.Format("ARHelper: OnError err: {0}, msg: {1}", err, msg));
        }

        public override void OnFirstRemoteVideoDecoded(RtcConnection connection, uint remoteUid, int width, int height, int elapsed)
        {
            Debug.LogWarningFormat("ARHelper: OnFirstRemoteVideoDecoded: uid:{0} w:{1} h:{2} elapsed:{3}", remoteUid, width, height, elapsed);
        }

        public override void OnJoinChannelSuccess(RtcConnection connection, int elapsed)
        {
            int build = 0;
            Debug.Log("ARHelper: Agora: OnJoinChannelSuccess ");
            Debug.Log(string.Format("ARHelper: sdk version: ${0}", _technicianManager.mRtcEngine.GetVersion(ref build)));
            Debug.Log(string.Format("ARHelper: sdk build: ${0}", build));
            Debug.Log(string.Format("ARHelper: OnJoinChannelSuccess channelName: {0}, uid: {1}, elapsed: {2}",
                                connection.channelId, connection.localUid, elapsed));
            
            _technicianManager.EnableSharing();
        }

        /*
        public override void OnRejoinChannelSuccess(RtcConnection connection, int elapsed)
        {
            Debug.Log("OnRejoinChannelSuccess");
        }

        public override void OnLeaveChannel(RtcConnection connection, RtcStats stats)
        {
            Debug.Log("OnLeaveChannel");
        }

        public override void OnClientRoleChanged(RtcConnection connection, CLIENT_ROLE_TYPE oldRole, CLIENT_ROLE_TYPE newRole, ClientRoleOptions newRoleOptions)
        {
            Debug.Log("OnClientRoleChanged");
        }*/

        public override void OnUserJoined(RtcConnection connection, uint uid, int elapsed)
        {
            Debug.Log(string.Format("ARHelper: OnUserJoined uid: ${0} elapsed: ${1}", uid, elapsed));
            
            GameObject go = GameObject.Find("myImage");
            if (go == null)
            {
                return;
            }

            VideoSurface videoSurface = go.GetComponent<VideoSurface>();
            if (videoSurface != null)
            {
                videoSurface.enabled = true;
                // configure videoSurface
                videoSurface.SetForUser(uid, _technicianManager.channelName);
                videoSurface.SetEnable(true);
                //videoSurface.
                //videoSurface.SetGameFps(30);
            }
            
            //TechnicianManager.MakeVideoView(uid, _technicianManager.channelName);
        }

        public override void OnUserOffline(RtcConnection connection, uint uid, USER_OFFLINE_REASON_TYPE reason)
        {
            // remove video stream
            Debug.Log(string.Format("ARHelper: OnUserOffLine uid: ${0}, reason: ${1}", uid, (int)reason));
            // this is called in main thread
            GameObject go = GameObject.Find(uid.ToString());
            if (!ReferenceEquals(go, null))
            {
                UnityEngine.Object.Destroy(go);
            }
            //TechnicianManager.DestroyVideoView(uid);
        }
        /*
        public override void OnUplinkNetworkInfoUpdated(UplinkNetworkInfo info)
        {
            Debug.Log("OnUplinkNetworkInfoUpdated");
        }

        public override void OnDownlinkNetworkInfoUpdated(DownlinkNetworkInfo info)
        {
            Debug.Log("OnDownlinkNetworkInfoUpdated");
        }*/
    }
    
    internal class VideoFrameObserver : IVideoFrameObserver
    {
        private readonly TechnicianManager _technicianManager;

        internal VideoFrameObserver(TechnicianManager technicianManager)
        {
            _technicianManager = technicianManager;
        }

        public override bool OnCaptureVideoFrame(VIDEO_SOURCE_TYPE type, VideoFrame videoFrame)
        {
            Debug.Log("OnCaptureVideoFrame-----------" + " width:" + videoFrame.width + " height:" +
                      videoFrame.height);
            //_agoraVideoRawData.VideoFrameWidth = videoFrame.width;
            //_agoraVideoRawData.VideoFrameHeight = videoFrame.height;
            //lock (_agoraVideoRawData.VideoBuffer)
            //{
            //    _agoraVideoRawData.VideoBuffer = videoFrame.yBuffer;
            //}
            return true;
        }

        public override bool OnRenderVideoFrame(string channelId, uint uid, VideoFrame videoFrame)
        {
            Debug.Log("OnRenderVideoFrameHandler-----------" + " uid:" + uid + " width:" + videoFrame.width +
                      " height:" + videoFrame.height);
            return true;
        }
    }
}