I'm working on a web application where I need to play two separate video tracks from a single MP4 file using Media Source Extensions (MSE) and MP4Box.js. I've set up MediaSource and SourceBuffer objects for each track and am using MP4Box.js to process the video file and extract the tracks. However, I'm encountering issues with playing the video tracks in the browser. The video elements don't display any content, and the SourceBuffer object shows no buffered data (buffered: TimeRanges { length: 0 }) even after appending the initialization segments and media segments.
I would appreciate any help understanding why the video isn't playing and how to correctly append and play the video tracks, as I think there is something that I am missing using the sourceBuffer from MediaSource.
I think I followed the standard approach for using MSE with mp4box.js. My script initializes a MediaSource object for each video element and creates a SourceBuffer for each. Using MP4Box.js, I fetch the video file and process the tracks. For each track, I append the initialization segment first, followed by the media segments to the respective SourceBuffer.
I added console logs statements throughout the script to monitor the process and ensure the segments are being appended. However, despite no errors being logged, the video elements remain blank, and the SourceBuffer.buffered.length remains at 0, indicating no data is buffered. I was expecting the video tracks to play in the respective video elements after appending the segments.
Additionally, I ensured the mimeCodec string matches the video file's format and codecs, and verified that the browser supports the specified MIME type. I'm looking for insights into what might be going wrong with the buffer appending or any steps I might have missed in setting up MSE with MP4Box.js for multiple video tracks.
So far, this is what I have came up with:
<!DOCTYPE html>
<html>
<head>
<title>Video tracks player</title>
<script src="https://gpac.github.io/mp4box.js/dist/mp4box.all.js"></script>
</head>
<body>
<video id="video1" controls width="500"></video>
<video id="video2" controls width="500"></video>
<script>
const video1 = document.getElementById('video1');
const video2 = document.getElementById('video2');
const assetURL = 'video.mp4'; // Replace with your video file path
function initVideoPlayer(video, trackIndex) {
console.log('Initializing video player', { video, trackIndex });
const mediaSource = new MediaSource();
video.src = URL.createObjectURL(mediaSource);
console.log('MediaSource and video source set', { mediaSource, videoSrc: video.src });
mediaSource.addEventListener('sourceopen', () => {
console.log('MediaSource opened', { readyState: mediaSource.readyState });
fetch(assetURL)
.then(response => {
console.log('Video file fetched', { assetURL, status: response.status });
return response.arrayBuffer();
})
.then(arrayBuffer => {
console.log('\n\nArrayBuffer obtained from fetch', { byteLength: arrayBuffer.byteLength });
const mp4boxFile = MP4Box.createFile();
console.log('MP4Box file created');
mp4boxFile.onReady = (info) => {
console.log('MP4Box file is ready', { info });
const videoTracks = info.tracks.filter(track => track.type === "video");
console.log('Video tracks filtered', { videoTracks });
if (videoTracks[trackIndex]) {
const track = videoTracks[trackIndex];
console.log('Processing track', { trackIndex, track });
const mime = `video/mp4; codecs="${track.codec}"`;
const sourceBuffer = mediaSource.addSourceBuffer(mime);
console.log('SourceBuffer created', { mime, sourceBuffer });
// Set segment options
mp4boxFile.setSegmentOptions(track.id, sourceBuffer, { nbSamples: track.nb_samples });
console.log('Segment options set', { trackId: track.id, nbSamples: track.nb_samples });
// Append initialization segments first
const initSegs = mp4boxFile.initializeSegmentation();
initSegs.forEach((initSeg) => {
if (initSeg.id === track.id) {
sourceBuffer.appendBuffer(initSeg.buffer);
console.log('Appended initialization segment', { initSeg }, initSeg.user.buffered);
}
});
sourceBuffer.addEventListener('updateend', () => {
console.log('updateend event after appending init segment', { sourceBuffer });
});
sourceBuffer.addEventListener('error', () => {
console.error('Error while appending init segment', { sourceBuffer });
});
// Handle media segments
mp4boxFile.onSegment = (id, user, buffer) => {
console.log('Received segment', { trackId: id, bufferByteLength: buffer.byteLength });
if (!sourceBuffer.updating) {
sourceBuffer.appendBuffer(buffer);
console.log('Appended media segment to SourceBuffer', { buffer });
}
};
mp4boxFile.start();
console.log('MP4Box file start called');
}
};
arrayBuffer.fileStart = 0;
mp4boxFile.appendBuffer(arrayBuffer);
console.log('Appended ArrayBuffer to MP4Box file', { fileStart: 0 });
})
.catch(error => {
console.error('Error while fetching video:', error);
});
});
}
initVideoPlayer(video1, 0); // Initialize first track
initVideoPlayer(video2, 1); // Initialize second track
</script>
</body>
</html>
As well as my console logs:
test.html:22 Initializing video player {video: video#video1, trackIndex: 0}
test.html:26 MediaSource and video source set {mediaSource: MediaSource, videoSrc: 'blob:http://127.0.0.1:5500/8b52067c-260e-4ec5-bf56-a25e06493e6d'}
test.html:22 Initializing video player {video: video#video2, trackIndex: 1}
test.html:26 MediaSource and video source set {mediaSource: MediaSource, videoSrc: 'blob:http://127.0.0.1:5500/8da1916c-ade5-4c8d-a97f-5de14c21e36f'}
test.html:29 MediaSource opened {readyState: 'open'}
test.html:29 MediaSource opened {readyState: 'open'}
test.html:33 Video file fetched {assetURL: 'video.mp4', status: 200}
test.html:33 Video file fetched {assetURL: 'video.mp4', status: 200}
test.html:31 Fetch finished loading: GET "http://127.0.0.1:5500/video.mp4".
(anonymous) @ test.html:31
test.html:31 Fetch finished loading: GET "http://127.0.0.1:5500/video.mp4".
(anonymous) @ test.html:31
test.html:37
ArrayBuffer obtained from fetch {byteLength: 8542253}
test.html:40 MP4Box file created
test.html:43 MP4Box file is ready {info: {…}}
test.html:46 Video tracks filtered {videoTracks: Array(2)}
test.html:50 Processing track {trackIndex: 0, track: {…}}
test.html:54 SourceBuffer created {mime: 'video/mp4; codecs="avc1.4d4028"', sourceBuffer: SourceBuffer}mime: "video/mp4; codecs=\"avc1.4d4028\""sourceBuffer: SourceBufferappendWindowEnd: InfinityappendWindowStart: 0buffered: TimeRanges {length: 0}mode: "segments"onabort: nullonerror: nullonupdate: nullonupdateend: nullonupdatestart: nulltimestampOffset: 0updating: false[[Prototype]]: SourceBuffer[[Prototype]]: Object
test.html:58 Segment options set {trackId: 1, nbSamples: 298}
test.html:65 Appended initialization segment {initSeg: {…}} TimeRanges {length: 0}
test.html:78 Received segment {trackId: 1, bufferByteLength: 5241615}
test.html:86 MP4Box file start called
test.html:92 Appended ArrayBuffer to MP4Box file {fileStart: 0}
test.html:37
ArrayBuffer obtained from fetch {byteLength: 8542253}
test.html:40 MP4Box file created
test.html:43 MP4Box file is ready {info: {…}}
test.html:46 Video tracks filtered {videoTracks: Array(2)}
test.html:50 Processing track {trackIndex: 1, track: {…}}
test.html:54 SourceBuffer created {mime: 'video/mp4; codecs="avc1.4d4028"', sourceBuffer: SourceBuffer}
test.html:58 Segment options set {trackId: 2, nbSamples: 290}
test.html:65 Appended initialization segment {initSeg: {…}} TimeRanges {length: 0}
test.html:78 Received segment {trackId: 2, bufferByteLength: 3355562}
test.html:86 MP4Box file start called
test.html:92 Appended ArrayBuffer to MP4Box file {fileStart: 0}
test.html:70 updateend event after appending init segment {sourceBuffer: SourceBuffer}
test.html:70 updateend event after appending init segment {sourceBuffer: SourceBuffer}