How to pick which MediaStreamTrack is currently playing?

Question

I have a MediaStream with 1 audio track and 2 video tracks. This stream is created by combining the audio and video track from navigator.mediaDevices.getUserMedia with the video track from navigator.mediaDevices.getDisplayMedia.

var camStream = await navigator.mediaDevices.getUserMedia({audio: true, video: true, width: "1280"});
var screenStream = await navigator.mediaDevices.getDisplayMedia({video: true, width: "1280"});
var screenTrack = screenStream.getVideoTracks()[0];

camStream.addTrack(screenTrack);

At this point, camStream has 1 audio track and 2 video tracks.

I have an HTML element <video id="local-video"></video> which I use to play the stream by setting the srcObject attribute:

document.getElementById("local-video").srcObject = camStream;

The problem is that I can't choose which track is being played. The camStream video track always plays over the the screenStream video track. Is there a way where I can keep both video tracks in the stream, but choose which one is currently playing in the video element?

So far the only thing that's worked is to remove one of the video tracks from the stream so that the other can play, but I'd like to keep both video tracks in the stream so that I can switch b/w them.

Answer 1

The MediaElement.videoTracks and MediaElement.audioTracks should be the general solution to this, but browser support is still not great as of today since no browser really supports this officially and none supports it with MediaStreams at all...

So below is a Snippet showing how it should be, but that's for the future.

const vid = document.querySelector("video");
const btn = document.querySelector("button");
const track1 = makeVideoTrack("red");
const track2 = makeVideoTrack("blue");

const stream = new MediaStream([track1, track2]);
vid.srcObject = stream;

btn.onclick = () => {
  // setting one track to selected will unselect the previously selected one
  // so we need to grab the current state to switch it
  const state = vid.videoTracks[0].selected;
  vid.videoTracks[1].selected = state;
  vid.videoTracks[0].selected = !state;
};

function makeVideoTrack(color) {
  const canvas = document.createElement("canvas");
  const ctx = canvas.getContext("2d");
  ctx.fillStyle = color;
  let x = 0;
  const anim = () => {
    x = (x + 1) % canvas.width;
    ctx.clearRect(0, 0, canvas.width, canvas.height);
    ctx.fillRect(x-5, 0, 10, canvas.height);
    requestAnimationFrame( anim );
  };
  anim();
  return canvas.captureStream().getVideoTracks()[0];
}

<video controls autoplay muted></video>

<button>switch video track</button>

For today, what can be done in your situation where you have access to the MediaStream, is to build a new MediaStream containing only the videoTrack you want. (One could be tempted to just reorder these tracks, but this would work only in current Chrome and Firefox and specs make it clear we should not rely on this behavior)

const vid = document.querySelector("video");
const btn = document.querySelector("button");
const tracks = [makeVideoTrack("red"), makeVideoTrack("blue")];
let selected_index = 0;
vid.srcObject = new MediaStream([ tracks[selected_index] ]);

btn.onclick = () => {
  selected_index = (selected_index + 1) % tracks.length
  vid.srcObject = new MediaStream([ tracks[selected_index] ]);
};

function makeVideoTrack(color) {
  const canvas = document.createElement("canvas");
  const ctx = canvas.getContext("2d");
  ctx.fillStyle = color;
  let x = 0;
  const anim = () => {
    x = (x + 1) % canvas.width;
    ctx.clearRect(0, 0, canvas.width, canvas.height);
    ctx.fillRect(x-5, 0, 10, canvas.height);
    requestAnimationFrame( anim );
  };
  anim();
  return canvas.captureStream().getVideoTracks()[0];
}

<video controls autoplay muted></video>

<button>switch video track</button>