Segments not loading in HLS Stream encoded with FFMPEG

Question

I'm currently facing a significant issue with my transcoding pipeline. Some video files produce a corrupted m3u8 playlist, wherein a few segments load but then the process halts. Even jumping to different segments in the video player doesn't resolve the issue. However, the file in question works fine when I use third-party transcoding services that utilize ffmpeg in the background, so the problem is definitely with my FFMPEG code.

To diagnose the issue, I've tried the following steps:

• Simplifying the FFMPEG command.
• Testing across different browsers, but the problem persists in every browser.
• Using various video players (It works with VLC, Bitmovin, Video.js) but not with Plyr, which I usually use. This could be because they ignore the error or are less sensitive to errors.
• My files are hosted on AWS S3, but I also tried streaming them via the Next.js public folder and a custom web server, so the issue is not related to hosting.
• Ruled out CORS as the issue.
• Attempted to detect errors using ffprobe, but found no results.
• Modified the bitrate in the playlist.
• Changed the viewing device.
• Altered the segment length, but the issue persists: with 10s segments, it's segment 2 that causes the problem; with 5s segments, then segment 4.
• No error output is visible in ffmpeg, even when using verbose mode.

mediastreamvalidator on Apple prints out error with provided bitrate and used bitrate but changing that also did not help.

HLS Analyzer shows a buffer stalled error.

const fs = require('fs');
const path = require('path');
const { v4: uuidv4 } = require('uuid');
require('dotenv').config();
const { processVideo } = require("./ffmpegHandler");


const VIDEO_PATH = process.env.VIDEO_PATH; 
const VIDEO_STATE = JSON.parse(process.env.VIDEO_STATE);
const VIDEO_SIZE = JSON.parse(process.env.VIDEO_SIZE);
const VIDEO_DURATION = process.env.VIDEO_DURATION;

const OUTPUT_DIRECTORY = '/tmp/output';

console.log("Video Path: " + VIDEO_PATH)
console.log("Video State: " + VIDEO_STATE)
console.log("Video Size: " + VIDEO_SIZE)
console.log("Video Duration: " + VIDEO_DURATION)

async function processLocalVideo(localPath) {
    const uniqueFolderName = uuidv4();  
    const localOutputDirectory = path.join(OUTPUT_DIRECTORY, uniqueFolderName);

    // Ensure the output directory exists
    if (!fs.existsSync(localOutputDirectory)) {
        fs.mkdirSync(localOutputDirectory, { recursive: true });
    }

    // Process the video locally
    processVideo(localPath, VIDEO_STATE, VIDEO_SIZE, VIDEO_DURATION, (err, message) => {
        if (err) {
            console.error('Error processing video:', err);
            return;
        }
        console.log('Video processing complete. HLS files are located at:', localOutputDirectory);
    });
}

// Start the processing
processLocalVideo(VIDEO_PATH).catch((error) => {
    console.error('An error occurred during video processing:', error);
});

const {exec} = require("child_process");
const ffmpegPath = require("ffmpeg-static");
const fs = require("fs");
const path = require("path");
const {spawn} = require("child_process");
const {generateArgs} = require("./generateArgs");
const {getResolutions, printDirectoryStructure, createMasterPlaylistContent} = require("./utility");


const OUTPUT_DIR = '/tmp/output';


function processVideo(filePath, imageState, size, duration, callback) {


    if (!fs.existsSync(OUTPUT_DIR)) {
        fs.mkdirSync(OUTPUT_DIR, {recursive: true});
    }

    const resolutions = getResolutions(size);
    const outputHLSPath = path.join(OUTPUT_DIR, 'dest.m3u8');
    const segmentPattern = 'segment_%03d.ts';
    const outputSegmentPath = path.join(OUTPUT_DIR, segmentPattern);

    const ffmpegPromises = resolutions.map(resolution => {
        return new Promise((resolve, reject) => {
            const scaleFilter = `scale=w=${resolution.width}:h=${resolution.height}`;
            const outputHLSPath = path.join(OUTPUT_DIR, resolution.label, 'dest.m3u8');
            const segmentPattern = `segment_%03d.ts`;
            const outputSegmentPath = path.join(OUTPUT_DIR, resolution.label, segmentPattern);

            // Ensure resolution-specific directory exists
            if (!fs.existsSync(path.join(OUTPUT_DIR, resolution.label))) {
                fs.mkdirSync(path.join(OUTPUT_DIR, resolution.label), {recursive: true});
            }

            const args = generateArgs(filePath, imageState, size, duration, resolution);


            const resolutionArgs = [
                ...args,
                '-hls_segment_filename', outputSegmentPath,
                outputHLSPath
            ];

            console.log(resolutionArgs)

            const command = `${ffmpegPath} ${resolutionArgs.join(" ")}`;


            const ffmpeg = spawn(ffmpegPath, resolutionArgs);


            ffmpeg.on('error', (error) => {
                console.error(`Error with FFmpeg process: ${error.message}`);
                reject(error);
            });

            ffmpeg.stderr.on('data', (data) => {
                console.error(`FFmpeg stderr: ${data}`);
            });

            ffmpeg.on('close', (code) => {
                if (code !== 0) {
                    reject(new Error(`FFmpeg process exited with code ${code}`));
                } else {
                    resolve();
                }
            });
        });
    });


    Promise.all(ffmpegPromises)
        .then(() => {
            const masterPlaylistContent = createMasterPlaylistContent(resolutions, OUTPUT_DIR);
            const masterPlaylistPath = path.join(OUTPUT_DIR, 'master.m3u8');
            fs.writeFileSync(masterPlaylistPath, masterPlaylistContent);

            printDirectoryStructure('/tmp/output', (err) => {
                if (err) {
                    console.error(err);
                    callback(err);
                } else {
                    console.log('Directory structure printed successfully.');
                    callback(null, 'All files processed successfully!');
                }
            });
        })
        .catch((error) => {
            callback(error);
        });
}


module.exports = {
    processVideo,
};

const {getColorMatrixFromColorMatrices, isNumber} = require("./utility");

function generateArgs(filePath, imageState, size, duration, resolution){
    const args = [];
    const filters = [];

    args.push("-i", filePath);


    const {
        flipX,
        flipY,
        rotation,
        crop,
        trim,
        gamma,
        colorMatrix,
        convolutionMatrix,
    } = imageState;

    console.log(crop);

    // 1. Flip
    if (flipX || flipY) {
        flipX && filters.push(`hflip`);
        flipY && filters.push(`vflip`);
    }

    // 2. Rotate
    if (rotation) {
        const width =
            Math.abs(size.width * Math.sin(rotation)) +
            Math.abs(size.height * Math.cos(rotation));

        const height =
            Math.abs(size.width * Math.cos(rotation)) +
            Math.abs(size.height * Math.sin(rotation));

        filters.push(
            `rotate='${rotation}:ow=${Math.floor(width)}:oh=${Math.floor(height)}'`
        );
    }

    // 3. Crop
    if (
        !(
            crop.x === 0 &&
            crop.y === 0 &&
            crop.width === size.width &&
            crop.height === size.height
        )
    ) {
        filters.push(`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`);
    }


    // 5. Convolution Matrix
    if (convolutionMatrix && convolutionMatrix.clarity) {
        filters.push(`convolution='${convolutionMatrix.clarity.join(" ")}'`);
    }

    // 6. Gamma
    if (gamma > 0) {
        filters.push(`eq=gamma=${gamma}:gamma_weight=0.85`);
    }

    // 7. Color Matrix
    const colorMatrices = Object.values(colorMatrix || {}).filter(Boolean);
    if (colorMatrices.length) {
        // See helper section below for the getColorMatrixFromColorMatrices function definition
        const colorMatrix = getColorMatrixFromColorMatrices(colorMatrices);
        const skip = [4, 9, 14, 19];
        const cl = colorMatrix;
        const ccm = colorMatrix.filter((v, i) => !skip.includes(i));
        const [ro, go, bo] = [cl[4] + cl[3], cl[9] + cl[8], cl[14] + cl[13]];
        filters.push(
            `colorchannelmixer=${ccm.join(":")}`,
            `lutrgb=r=val+(${ro * 255}):g=val+(${go * 255}):b=val+(${bo * 255})`
        );
    }


    if (resolution) {
        const scaleFilter = `scale=w=${resolution.width}:h=${resolution.height}`;
        filters.push(scaleFilter);
    }


    // 8. Trim
    if (trim) {
        const inputRanges = (
            Array.isArray(trim) && isNumber(trim[0]) ? [trim] : trim
        )
            .map((range, index) => {
                const from = range[0] * duration;
                const to = range[1] * duration;
                const v = `[0:v]trim=start=${from}:end=${to},setpts=PTS-STARTPTS${filters
                    .map((filter) => "," + filter)
                    .join("")}[${index}v];`;
                const a = `[0:a]atrim=start=${from}:end=${to},asetpts=PTS-STARTPTS[${index}a];`;
                return v + a;
            })
            .join("");


        filters.length = 0;

        const inputRangesKeys = trim
            .map((_, index) => `[${index}v][${index}a]`)
            .join("");

        const concatOutput = `${inputRangesKeys}concat=n=${trim.length}:v=1:a=1[outv][outa]`;

        args.push("-filter_complex", `${inputRanges}${concatOutput}`);
        args.push("-map", "[outv]", "-map", "[outa]");
    } else {
        filters.length && args.push("-filter_complex", `${filters.join(",")}`);
    }


    // Add output file
    args.push(
        "-codec:v",
        "libx264",
        "-crf",
        "21",
        "-preset",
        "veryfast",
        "-g",
        "30",
        "-sc_threshold",
        "0",
        "-hls_time",
        "10",
        "-hls_list_size",
        "0",

    );

    return args
}

module.exports = {
    generateArgs,
};

const {exec} = require("child_process");


function isNumber(value) {
    return typeof value === "number" && !isNaN(value);
}

function getColorMatrixFromColorMatrices(colorMatrices) {
    return colorMatrices.length
        ? colorMatrices.reduce(
            (previous, current) => dotColorMatrix([...previous], current),
            colorMatrices.shift()
        )
        : [];
}

function dotColorMatrix(a, b) {
    const res = new Array(20);

    // R
    res[0] = a[0] * b[0] + a[1] * b[5] + a[2] * b[10] + a[3] * b[15];
    res[1] = a[0] * b[1] + a[1] * b[6] + a[2] * b[11] + a[3] * b[16];
    res[2] = a[0] * b[2] + a[1] * b[7] + a[2] * b[12] + a[3] * b[17];
    res[3] = a[0] * b[3] + a[1] * b[8] + a[2] * b[13] + a[3] * b[18];
    res[4] = a[0] * b[4] + a[1] * b[9] + a[2] * b[14] + a[3] * b[19] + a[4];

    // G
    res[5] = a[5] * b[0] + a[6] * b[5] + a[7] * b[10] + a[8] * b[15];
    res[6] = a[5] * b[1] + a[6] * b[6] + a[7] * b[11] + a[8] * b[16];
    res[7] = a[5] * b[2] + a[6] * b[7] + a[7] * b[12] + a[8] * b[17];
    res[8] = a[5] * b[3] + a[6] * b[8] + a[7] * b[13] + a[8] * b[18];
    res[9] = a[5] * b[4] + a[6] * b[9] + a[7] * b[14] + a[8] * b[19] + a[9];

    // B
    res[10] = a[10] * b[0] + a[11] * b[5] + a[12] * b[10] + a[13] * b[15];
    res[11] = a[10] * b[1] + a[11] * b[6] + a[12] * b[11] + a[13] * b[16];
    res[12] = a[10] * b[2] + a[11] * b[7] + a[12] * b[12] + a[13] * b[17];
    res[13] = a[10] * b[3] + a[11] * b[8] + a[12] * b[13] + a[13] * b[18];
    res[14] = a[10] * b[4] + a[11] * b[9] + a[12] * b[14] + a[13] * b[19] + a[14];

    // A
    res[15] = a[15] * b[0] + a[16] * b[5] + a[17] * b[10] + a[18] * b[15];
    res[16] = a[15] * b[1] + a[16] * b[6] + a[17] * b[11] + a[18] * b[16];
    res[17] = a[15] * b[2] + a[16] * b[7] + a[17] * b[12] + a[18] * b[17];
    res[18] = a[15] * b[3] + a[16] * b[8] + a[17] * b[13] + a[18] * b[18];
    res[19] = a[15] * b[4] + a[16] * b[9] + a[17] * b[14] + a[18] * b[19] + a[19];

    return res;
}

function getResolutions(videoSize) {
    // Define the resolutions
    const resolutions = [
        {width: 1280, height: 720, label: "720p", bandwidth: "1400000"},
        {width: 854, height: 480, label: "480p", bandwidth: "800000"},
        {width: 426, height: 240, label: "240p", bandwidth: "600000"}
    ];

    // Filter out resolutions that are larger than the video's original size
    return resolutions.filter(res => res.width <= videoSize.width);
}


function printDirectoryStructure(directory, callback) {
    exec(`find '${directory}' -print`, (error, stdout, stderr) => {
        if (error) {
            console.error(`exec error: ${error}`);
            return callback(error);
        }
        if (stderr) {
            console.error(`stderr: ${stderr}`);
            return callback(stderr);
        }
        console.log(`Directory structure of ${directory}:\n${stdout}`);
        callback(null);
    });
}


function createMasterPlaylistContent(resolutions, outputDir) {
    let content = '#EXTM3U\n#EXT-X-VERSION:3\n';
    resolutions.forEach((resolution) => {
        content += `#EXT-X-STREAM-INF:BANDWIDTH=${resolution.bandwidth},RESOLUTION=${resolution.width}x${resolution.height}\n`;
        content += `${resolution.label}/dest.m3u8\n`;
    });
    return content;
}


function getKeyFromUrl(url) {
    const urlParts = new URL(url);
    return urlParts.pathname.substring(1);
}


module.exports = {
    getResolutions,
    getColorMatrixFromColorMatrices,
    isNumber,
    printDirectoryStructure,
    createMasterPlaylistContent,
    getKeyFromUrl
};