react-native-webrtc not showing the remote stream

Question

Hey I am trying to create a video calling functionality, in my code I have done everything (as of my knowledge of WebRTC, cause I am just 5 days into WebRTC) like exchanging offer and answer, exchanging candidates, but for some reason I am not being able to see the remote stream (other party video and audio) and also not even getting any errors, here I have 3 screens CallingScreen (where the call happens and 2 peers communicate normally), OutCallScreen (where one user creates an offer and sends it to the other party), InCallScreen (where the user gets notified that he is getting an incoming call and creates answer and sends it back to the offer owner), and I also have the WebSocket server where I get the offer, answer,candidates and send them to there respective destinations, please check these and let me know where I am missing in these, that I am not being able to see the remote stream.

I Hope giving all of these 4 full files might help catching the problem

CallingScreen.js

import { StyleSheet, Text, View, Pressable, Image } from 'react-native';
import React, { useState, useRef, useEffect } from 'react';
import { Height, Width } from '../utils';
import { moderateScale } from 'react-native-size-matters';
import { ScreenCapturePickerView, RTCPeerConnection, RTCIceCandidate, RTCSessionDescription, RTCView, MediaStream, MediaStreamTrack, mediaDevices, registerGlobals } from 'react-native-webrtc';
import { setDoc, collection, doc, getDocs, onSnapshot } from 'firebase/firestore';
import { FIREBASE_DB, FIREBASE_AUTH } from '../firebaseConfig';
import { encryptData, decryptData } from '../EncryptData'
import AsyncStorage from '@react-native-async-storage/async-storage';
import ws from './WebSocketConn';
import { peerConnection } from '../CallingUtils';

import Constants from 'expo-constants';
const SECRET_KEY = Constants.expoConfig.extra.SECRET_KEY;

const peerConstraints = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] };

const CallingScreen = ({ navigation, route }) => {
  const [localMediaStream, setLocalMediaStream] = useState(null);
  const [remoteMediaStream, setRemoteMediaStream] = useState(null);
  const [isVoiceOnly, setIsVoiceOnly] = useState(false);
  const { DocId, answer, recipientId } = route.params;


  useEffect(() => {
    getMediaStream();
    let localDescriptionSet = false;
  }, []);

  const getMediaStream = async () => {
    const mediaConstraints = {
      audio: true,
      video: true,
    };
    try {
      const stream = await mediaDevices.getUserMedia(mediaConstraints);
      setLocalMediaStream(stream);

    } catch (error) {
      console.error('Error getting media stream:', error);
    }
  };

  const hangup = async () => {
    try {
      if (peerConnection) {
        peerConnection.close();
      }
      if (localMediaStream) {
        localMediaStream.getTracks().forEach(track => track.stop());
      }
    } catch (error) {
      console.error('Error hanging up:', error);
    } finally {
      setLocalMediaStream(null);
      setRemoteMediaStream(null);
    }
  };

  const handleIceCandidates = async () => {
    try {

      localMediaStream.getTracks().forEach(track => {
        peerConnection.addTrack(track, localMediaStream);
      });

      peerConnection.onicecandidate = async (event) => {
        if (event.candidate) {
          console.log('sending the candidate to', recipientId, 'by', CustomUUID);
          const idToken = await FIREBASE_AUTH.currentUser.getIdToken();
          const encryptedIdToken = encryptData(idToken, SECRET_KEY);
          const CustomUUID = await AsyncStorage.getItem('CustomUUID');
          console.log('my id is', CustomUUID, 'and i am sending candidate to', recipientId);
          const candidateMessage = {
            type: 'candidate',
            userId: CustomUUID,
            recipientId: recipientId,
            idToken: encryptedIdToken,
            candidate: event.candidate
          };
          const candidateMessageString = JSON.stringify(candidateMessage);
          ws.send(candidateMessageString);
        }
      };

      ws.onmessage = async (event) => {
        console.log('candidate, being set in CallingScreen');
        const message = JSON.parse(event.data);
        if (message.type === 'candidate') {
          try {
            const candidate = new RTCIceCandidate(message.candidate);
            await peerConnection.addIceCandidate(candidate);
          } catch (error) {
            console.error('Error handling ICE candidate:', error);
          }
        }
      };

      peerConnection.ontrack = (event) => {
        const remoteStream = event.streams[0];
        setRemoteMediaStream(remoteStream);
      };

    } catch (error) {
      console.error('Error handling incoming answer:', error);
    }
  }


  useEffect(() => {
    if (answer && recipientId && localMediaStream) {
      handleIceCandidates();
    }
  }, [answer])



  return (
    <View style={styles.Container}>
      <View style={styles.CallingTop}>

        {localMediaStream ? <RTCView mirror={true} objectFit={'cover'} streamURL={localMediaStream.toURL()} zOrder={0} style={styles.CallingTopImage} />
          : <Image source={require('../assets/Images/call1.jpg')} style={styles.CallingTopImage} />}

        <View style={styles.CallingTopNameView}>
          <Image source={require('../assets/Images/CallingNameShadow.png')} style={styles.CallingTopNameShadowImage} />
          <View style={styles.CallerDetails}>
            <Image source={require('../assets/Images/call1.jpg')} style={styles.CallerDetailsImage} />
            <Text style={styles.CallerDetailsName}>Yung-Chen</Text>
          </View>
        </View>
      </View>
      <View style={styles.CallingBottom}>

        {remoteMediaStream ? <RTCView mirror={true} objectFit={'cover'} streamURL={remoteMediaStream.toURL()} zOrder={0} style={styles.CallingBottomImage} />
          : <Image source={require('../assets/Images/call2.jpg')} style={styles.CallingBottomImage} />}

        <View style={styles.CallingBottomCallOptions}>
          <Pressable style={styles.CallOptionsButton}>
            <Image source={require('../assets/Icons/CallMuteIcon.png')} style={styles.CallOptionsButtonImage} />
          </Pressable>
          <Pressable style={styles.CallOptionsButton}>
            <Image source={require('../assets/Icons/CallMicIcon.png')} style={styles.CallOptionsButtonImage} />
          </Pressable>
          <Pressable style={styles.CallOptionsButton}>
            <Image source={require('../assets/Icons/CallCameraOption.png')} style={styles.CallOptionsButtonImage} />
          </Pressable>
          <Pressable onPress={() => { hangup() }} style={styles.CallHangUpOptionsButton}>
            <Image source={require('../assets/Icons/CallHangUpIcon.png')} style={styles.CallHangUpOptionsButtonImage} />
          </Pressable>
        </View>
      </View>
    </View>
  )
}

export default CallingScreen

OutCallingScreen.js

import { StyleSheet, Text, View, Pressable, Image } from 'react-native';
import React, { useState, useRef, useEffect } from 'react';
import { Height, Width } from '../utils';
import { moderateScale } from 'react-native-size-matters';
import { ScreenCapturePickerView, RTCPeerConnection, RTCIceCandidate, RTCSessionDescription, RTCView, MediaStream, MediaStreamTrack, mediaDevices, registerGlobals } from 'react-native-webrtc';
import { setDoc, addDoc, collection, doc, getDocs, onSnapshot, Timestamp } from 'firebase/firestore';
import { FIREBASE_DB, FIREBASE_AUTH } from '../firebaseConfig';
import AsyncStorage from '@react-native-async-storage/async-storage';
import { encryptData, decryptData } from '../EncryptData'
import ws from './WebSocketConn';
import { peerConnection } from '../CallingUtils';

import Constants from 'expo-constants';
const SECRET_KEY = Constants.expoConfig.extra.SECRET_KEY;

const peerConstraints = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] };


const OutCallScreen = ({ navigation, route }) => {

  const [localMediaStream, setLocalMediaStream] = useState(null);
  const [CustomUUID, setCustomUUID] = useState(null);
  const { chatId } = route.params;

  AsyncStorage.getItem('CustomUUID').then((CustomUUID) => {
    setCustomUUID(CustomUUID);
  });

  useEffect(() => {
    getMediaStream();

    return () => {
      if (peerConnection) {
        peerConnection.close();
      }
      if (localMediaStream) {
        localMediaStream.getTracks().forEach(track => track.stop());
      }
    };
  }, []);

  const getMediaStream = async () => {
    const mediaConstraints = {
      audio: true,
      video: true,
    };
    try {
      const stream = await mediaDevices.getUserMedia(mediaConstraints);
      setLocalMediaStream(stream);
    } catch (error) {
      console.error('Error getting media stream:', error);
    }
  };

  const createOffer = async () => {
    try {
      localMediaStream.getTracks().forEach(track => {
        peerConnection.addTrack(track, localMediaStream);
      });

      const offerDescription = await peerConnection.createOffer();
      await peerConnection.setLocalDescription(offerDescription);

      const idToken = await FIREBASE_AUTH.currentUser.getIdToken();
      const encryptedIdToken = encryptData(idToken, SECRET_KEY);
      console.log('sending offer to', CustomUUID === chatId.split('_')[0] ? chatId.split('_')[1] : chatId.split('_')[0], 'by', CustomUUID);
      const offerStructure = {
        senderId: CustomUUID,
        recipientId: CustomUUID === chatId.split('_')[0] ? chatId.split('_')[1] : chatId.split('_')[0],
        status: 'pending',
        timestamp: Timestamp.now(),
        offerDescription: offerDescription,
      }
      const encryptedOfferStructure = encryptData(JSON.stringify(offerStructure), SECRET_KEY);

      ws.send(JSON.stringify({
        type: 'offer',
        userId: CustomUUID,
        idToken: encryptedIdToken,
        offer: encryptedOfferStructure
      }));

    } catch (error) {
      console.error('Error creating offer:', error);
    }
  };


  useEffect(() => {
    if (chatId && CustomUUID && localMediaStream) {
      createOffer();
    }
    ws.onmessage = async (event) => {
      const parsedMessage = JSON.parse(event.data);
      if (parsedMessage.type === 'answer') {
        const answerDescription = new RTCSessionDescription(parsedMessage.answer);
        await peerConnection.setRemoteDescription(answerDescription);
        console.log('got the answer from ', parsedMessage.userId);
        navigation.navigate('CallingScreen', { answer: parsedMessage.answer, recipientId: parsedMessage.userId });
      }
    }
  }, [chatId, CustomUUID, localMediaStream]);



  return (
    <View style={styles.Container}>
      {localMediaStream ? <RTCView mirror={true} objectFit={'cover'} streamURL={localMediaStream.toURL()} zOrder={0} style={styles.CallingTopImage} />
        : <Image source={require('../assets/Images/call1.jpg')} style={styles.CallingTopImage} />}
      <View style={styles.backgroundShadowCover}>
        <View style={styles.recipientInfo}>
          <Image source={require('../assets/Images/call1.jpg')} style={styles.recipientImage} />
          <Text style={styles.recipientName}>Mei Ling</Text>
          <Text style={styles.recipientCallTime}>Connecting</Text>
        </View>
        <View style={styles.CallingOptions}>
          <Pressable style={styles.VideoToggleButton}>
            <Image source={require('../assets/Icons/CallCameraOption.png')} style={styles.VideoToggleButtonImage} />
          </Pressable>
          <Pressable style={styles.hangUpButton}>
            <Image source={require('../assets/Icons/CallHangUpIcon.png')} style={styles.hangUpButtonImage} />
          </Pressable>
          <Pressable style={styles.MicToggleButton}>
            <Image source={require('../assets/Icons/CallMicIcon.png')} style={styles.MicToggleButtonImage} />
          </Pressable>
        </View>
      </View>
    </View>
  )
}

export default OutCallScreen

InCallScreen.js

import { StyleSheet, Text, View, Pressable, Image } from 'react-native';
import React, { useState, useRef, useEffect } from 'react';
import { Height, Width } from '../utils';
import { moderateScale } from 'react-native-size-matters';
import { ScreenCapturePickerView, RTCPeerConnection, RTCIceCandidate, RTCSessionDescription, RTCView, MediaStream, MediaStreamTrack, mediaDevices, registerGlobals } from 'react-native-webrtc';
import { setDoc, collection, doc, getDocs, onSnapshot } from 'firebase/firestore';
import { FIREBASE_DB, FIREBASE_AUTH } from '../firebaseConfig';
import { encryptData, decryptData } from '../EncryptData'
import AsyncStorage from '@react-native-async-storage/async-storage';
import ws from './WebSocketConn';
import { peerConnection } from '../CallingUtils';


import Constants from 'expo-constants';
const SECRET_KEY = Constants.expoConfig.extra.SECRET_KEY;

const peerConstraints = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] };

const InCallScreen = ({ navigation, route }) => {

  const [localMediaStream, setLocalMediaStream] = useState(null);
  const { offer, DocId } = route.params;

  useEffect(() => {
    getMediaStream();

    return () => {
      if (localMediaStream) {
        localMediaStream.getTracks().forEach(track => track.stop());
      }
    };
  }, []);

  const getMediaStream = async () => {
    const mediaConstraints = {
      audio: true,
      video: true,
    };
    try {
      const stream = await mediaDevices.getUserMedia(mediaConstraints);
      setLocalMediaStream(stream);
    } catch (error) {
      console.error('Error getting media stream:', error);
    }
  };

  const AccpetCall = async () => {
    try {
      const idToken = await FIREBASE_AUTH.currentUser.getIdToken();
      const encryptedIdToken = encryptData(idToken, SECRET_KEY);
      const response = await fetch(`http://10.0.2.2:5000/users/UpdateCallStatus`, {
        method: 'PUT',
        credentials: 'include',
        headers: {
          'Authorization': 'Bearer ' + encryptedIdToken,
          'Accept': 'application/json',
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({ DocId: DocId, status: 'Accepted' })
      })
      const data = await response.json();
      if (response.status == 200) {
        createAnswer()
      }
    } catch (error) {
      console.error('error, updating call status, and creating answer:', error)
    }
  }

  const createAnswer = async () => {
    try {
      if (!peerConnection) {
        console.error('Peer connection is not initialized');
        return;
      }

      localMediaStream.getTracks().forEach(track => {
        peerConnection.addTrack(track, localMediaStream);
      });

      // Set remote description from the offer
      const offerDescription = new RTCSessionDescription(offer.offerDescription);
      await peerConnection.setRemoteDescription(offerDescription);

      // Create answer and set it as local description
      const answer = await peerConnection.createAnswer();
      await peerConnection.setLocalDescription(answer);

      // Get CustomUUID
      const CustomUUID = await AsyncStorage.getItem('CustomUUID');

      // Get idToken and encrypt it
      const idToken = await FIREBASE_AUTH.currentUser.getIdToken();
      const encryptedIdToken = encryptData(idToken, SECRET_KEY);

      ws.onmessage = async (event) => {
        console.log('candidate, being set in InCallScreen');
        const message = JSON.parse(event.data);
        if (message.type === 'candidate') {
          try {
            const candidate = new RTCIceCandidate(message.candidate);
            await peerConnection.addIceCandidate(candidate);
          } catch (error) {
            console.error('Error handling ICE candidate:', error);
          }
        }
      };

      console.log(offer.senderId);
      // Send ICE candidates to the offering peer
      peerConnection.onicecandidate = async (event) => {
        console.log('candidate, being sended by InCallScreen');
        console.log('sending candidate by', CustomUUID, 'to', offer.senderId);
        if (event.candidate) {
          const candidateMessage = {
            type: 'candidate',
            userId: CustomUUID,
            recipientId: offer.senderId,
            idToken: encryptedIdToken,
            candidate: event.candidate
          };
          const candidateMessageString = JSON.stringify(candidateMessage);
          ws.send(candidateMessageString);
        }
      };

      // Send the answer message to the offering peer
      const answerMessage = {
        type: 'answer',
        userId: CustomUUID,
        recipientId: offer.senderId,
        idToken: encryptedIdToken,
        answer: answer
      };
      console.log('sending answer by', CustomUUID, 'to', offer.senderId);
      const answerMessageString = JSON.stringify(answerMessage);
      ws.send(answerMessageString);

      // Navigate to the CallingScreen with offer and DocId
      navigation.navigate('CallingScreen', { answer: answer, DocId: DocId });
    } catch (error) {
      console.error('Error creating answer and sending it back:', error);
    }
  }

  const RejectCall = async () => {
    try {
      const idToken = await FIREBASE_AUTH.currentUser.getIdToken();
      const encryptedIdToken = encryptData(idToken, SECRET_KEY);
      const response = await fetch(`http://10.0.2.2:5000/users/UpdateCallStatus`, {
        method: 'PUT',
        credentials: 'include',
        headers: {
          'Authorization': 'Bearer ' + encryptedIdToken,
          'Accept': 'application/json',
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({ DocId: DocId, status: 'Rejected' })
      })
      const data = await response.json();
      if (response.status == 200) {
        navigation.goBack();
      }
    } catch (error) {
      console.error('error, updating call status:', error)
    }
  }


  return (
    <View style={styles.Container}>
      {localMediaStream ? <RTCView mirror={true} objectFit={'cover'} streamURL={localMediaStream.toURL()} zOrder={0} style={styles.CallerImage} />
        : <Image source={require('../assets/Images/call1.jpg')} style={styles.CallerImage} />}
      {/* <Image source={require('../assets/Icons/IncomingCallShadowCover.png')} style={styles.backgroundShadowCover} /> */}
      <View style={styles.backgroundShadowCover} />
      <View style={styles.ContentContainer}>
        <View style={styles.CallerInfo}>
          <Text style={styles.CallTypeText}>Incoming Call</Text>
          <Text style={styles.CallerName}>Mei-Ling</Text>
        </View>
        <View style={styles.CallOptions}>
          <Pressable style={styles.RejectCallButton}>
            <Image onPress={RejectCall} source={require('../assets/Icons/CallHangUpIcon.png')} style={styles.RejectCallButtonImage} />
          </Pressable>
          <Pressable onPress={AccpetCall} style={styles.AcceptCallButton}>
            <Image source={require('../assets/Icons/CallHangUpIcon.png')} style={styles.AcceptCallButtonImage} />
          </Pressable>
        </View>
      </View>
    </View>
  )
}

export default InCallScreen

server.js

const express = require('express');
const bodyParser = require('body-parser');
const cookieParser = require('cookie-parser');
const routes = require('./routes/route');
const admin = require('./firebaseAdminSDK');
const essentials = require('./essentials');
const http = require('http');
const WebSocket = require('ws');

require('dotenv-safe').config();

const app = express();
const server = http.createServer(app);
const wss = new WebSocket.Server({ server });

const { SECRET_KEY } = process.env;
const AUTH = admin.auth();
const DB = admin.firestore();
const STORAGE = admin.storage();

const PORT = process.env.PORT || 5000;

const userConnections = new Map();

app.use(bodyParser.json({ limit: '10mb' }));
app.use(bodyParser.urlencoded({ limit: '10mb', extended: true }));
app.use(cookieParser());

app.use('/', routes); app.listen(PORT, () => {
  console.log('Server is listening on Port:', PORT);
});

server.listen(8000, () => {
  console.log('websocket server started on port 8000');
});


// Websocket Code
wss.on('connection', async (ws, req) => {
  let userId;

  ws.on('message', async (message) => {
    const { userId: messageUserId, idToken, recipientId, type, offer, answer, candidate } = JSON.parse(message);
    try {
      const decryptedIdToken = essentials.decryptData(idToken, SECRET_KEY);
      const decodedToken = await AUTH.verifyIdToken(decryptedIdToken);
      if (decodedToken.uid) {
        userConnections.set(messageUserId, ws);
        userId = messageUserId;
      } else {
        ws.send('Unauthorized');
        return;
      }
    } catch (error) {
      console.error('Error handling message:', error);
      ws.send('Unauthorized');
    }




    try {
      if (type === 'answer') {
        console.log('sending back the answer to offer ownwer', recipientId)
        if (recipientId) {
          const connection = userConnections.get(recipientId);
          if (connection) {
            connection.send(JSON.stringify({ type: 'answer', answer: answer, userId: messageUserId }));
          }
        }
      }
    } catch (error) {
      console.error('Error sending answer:', error);
      ws.send('Error sending answer');
    }




    try {
      if (type === 'candidate') {
        if (recipientId && candidate) {
          const recipientConnection = userConnections.get(recipientId);
          if (recipientConnection) {
            recipientConnection.send(JSON.stringify({ type: 'candidate', candidate: candidate }));
          }
        }
      }
    } catch (error) {
      console.error('Error sending answer:', error);
      ws.send('Error sending answer');
    }




    try {
      if (type === 'offer') {
        const decryptedOffer = essentials.decryptData(offer, SECRET_KEY);
        const docRef = await DB.collection('calls').add(JSON.parse(decryptedOffer));
        const recipientUserId = JSON.parse(decryptedOffer).recipientId;
        const notification = {
          type: 'incoming_call',
          docId: docRef.id,
          offer: offer,
        };
        sendCallNotification(recipientUserId, notification);
      }
    } catch (error) {
      console.error('Error handling offer:', error);
      ws.send('Error Creating Offer');
    }
  });




  ws.on('close', () => {
    if (userId) {
      userConnections.delete(userId);
    }
  });
});


function sendCallNotification(recipientUserId, notification) {
  const connection = userConnections.get(recipientUserId);
  if (connection) {
    connection.send(JSON.stringify(notification));
  }
}