How to merge audio with video without losing original sound of video?

Question

My goal is to merge audio (mp3 music) with video captured by iPhone camera, I am able to merge audio with video using AVMutableComposition but in the final output the video sound is not coming.

Below is the code I am using:

open func mergeVideoWithAudio(videoUrl: URL, audioUrl: URL){
    
    let mixComposition: AVMutableComposition = AVMutableComposition()
    var mutableCompositionVideoTrack: [AVMutableCompositionTrack] = []
    var mutableCompositionAudioTrack: [AVMutableCompositionTrack] = []
    let totalVideoCompositionInstruction : AVMutableVideoCompositionInstruction = AVMutableVideoCompositionInstruction()
    
    let aVideoAsset: AVAsset = AVAsset(url: videoUrl)
    let aAudioAsset: AVAsset = AVAsset(url: audioUrl)
    
    if let videoTrack = mixComposition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid), let audioTrack = mixComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) {
        mutableCompositionVideoTrack.append(videoTrack)
        mutableCompositionAudioTrack.append(audioTrack)
    }
    
    let time = CMTimeMakeWithSeconds(Float64(musicTrimmerController.currentPlayerPosition), 1000)
    
    if let aVideoAssetTrack: AVAssetTrack = aVideoAsset.tracks(withMediaType: .video).first,
        let aAudioAssetTrack: AVAssetTrack = aAudioAsset.tracks(withMediaType: .audio).first {
        do {
            try mutableCompositionVideoTrack.first?.insertTimeRange(CMTimeRangeMake(kCMTimeZero, aVideoAssetTrack.timeRange.duration), of: aVideoAssetTrack, at: kCMTimeZero)
            
            try mutableCompositionAudioTrack.first?.insertTimeRange(CMTimeRangeMake(time, aVideoAssetTrack.timeRange.duration), of: aAudioAssetTrack, at: kCMTimeZero)
            
        } catch{
            print(error)
        }
        
        totalVideoCompositionInstruction.timeRange = CMTimeRangeMake(kCMTimeZero,aVideoAssetTrack.timeRange.duration)
        
        let compositionV = mixComposition.tracks(withMediaType: AVMediaType.video).last
        if ((aVideoAssetTrack != nil) && (compositionV != nil)) {
            compositionV?.preferredTransform = (aVideoAssetTrack.preferredTransform)
        }
    }
    if let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true).first {
        let outputURL = URL(fileURLWithPath: documentsPath).appendingPathComponent("movie.mov")
        do {
            if FileManager.default.fileExists(atPath: outputURL.path) {
                try FileManager.default.removeItem(at: outputURL)
            }
        } catch { }
        if let exportSession = AVAssetExportSession(asset: mixComposition, presetName: AVAssetExportPresetHighestQuality) {0
            exportSession.outputURL = outputURL
            exportSession.outputFileType = AVFileType.mp4
            exportSession.shouldOptimizeForNetworkUse = true
            
            /// try to export the file and handle the status cases
            exportSession.exportAsynchronously(completionHandler: {
                switch exportSession.status {
                case .failed:
                    print(exportSession.error as Any)
                case .cancelled:
                    print(exportSession.error as Any)
                default:
                    print("Save video output")
                }
            })
        }
    }
}

Answer 1

func mergeAudioWithVideo(audioURL: URL, videoURL: URL, completion: @escaping (URL?, Error?) -> Void) {

        let mixComposition: AVMutableComposition = AVMutableComposition()
        var mutableCompositionVideoTrack: [AVMutableCompositionTrack] = []
        var mutableCompositionAudioTrack: [AVMutableCompositionTrack] = []
        var mutableCompositionAudioOfVideoTrack: [AVMutableCompositionTrack] = []
        let totalVideoCompositionInstruction: AVMutableVideoCompositionInstruction = AVMutableVideoCompositionInstruction()

        let aVideoAsset: AVAsset = AVAsset(url: videoURL)
        let aAudioAsset: AVAsset = AVAsset(url: audioURL)

    mutableCompositionVideoTrack.append(mixComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: kCMPersistentTrackID_Invalid)!)
    mutableCompositionAudioTrack.append(mixComposition.addMutableTrack(withMediaType: AVMediaType.audio, preferredTrackID: kCMPersistentTrackID_Invalid)!)
    mutableCompositionAudioOfVideoTrack.append(mixComposition.addMutableTrack(withMediaType: AVMediaType.audio, preferredTrackID: kCMPersistentTrackID_Invalid)!)

    let aAudioOfVideoTrack: AVAssetTrack = aVideoAsset.tracks(withMediaType: AVMediaType.audio)[0]
             let aVideoAssetTrack: AVAssetTrack = aVideoAsset.tracks(withMediaType: .video)[0]
    let aAudioAssetTrack: AVAssetTrack = aAudioAsset.tracks(withMediaType: AVMediaType.audio)[0]

        do {
            try mutableCompositionAudioOfVideoTrack[0].insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: aVideoAssetTrack.timeRange.duration), of: aAudioOfVideoTrack, at: CMTime.zero)

            try mutableCompositionVideoTrack[0].insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: aVideoAssetTrack.timeRange.duration), of: aVideoAssetTrack, at: CMTime.zero)
            try mutableCompositionAudioTrack[0].insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: aVideoAssetTrack.timeRange.duration), of: aAudioAssetTrack, at: CMTime.zero)
        } catch {

        }

    totalVideoCompositionInstruction.timeRange = CMTimeRangeMake(start: CMTime.zero, duration: aVideoAssetTrack.timeRange.duration)

        let mutableVideoComposition: AVMutableVideoComposition = AVMutableVideoComposition()
    mutableVideoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)

        mutableVideoComposition.renderSize = CGSize(width: 1280, height: 720)//CGSize(1280,720)

        //find your video on this URl
        let savePathUrl: NSURL = NSURL(fileURLWithPath: NSHomeDirectory() + "/Documents/newVideo.mp4")

        do { // delete old video
            try FileManager.default.removeItem(at: savePathUrl as URL)
        } catch {
            print(error.localizedDescription)
        }

        let assetExport: AVAssetExportSession = AVAssetExportSession(asset: mixComposition, presetName: AVAssetExportPresetHighestQuality)!
    assetExport.outputFileType = AVFileType.mp4
        assetExport.outputURL = savePathUrl as URL
        assetExport.shouldOptimizeForNetworkUse = true

        assetExport.exportAsynchronously {
            switch assetExport.status {
            case AVAssetExportSession.Status.completed:

                //Uncomment this if u want to store your video in asset
                let assetsLib = ALAssetsLibrary()
                assetsLib.writeVideoAtPath(toSavedPhotosAlbum: savePathUrl as URL, completionBlock: nil)


                print("successsuccesssuccesssuccesssuccesssuccesssuccesssuccesssuccess")
            case AVAssetExportSession.Status.failed:
                print("failed \(assetExport.error)")
            case AVAssetExportSession.Status.cancelled:
                print("cancelled \(assetExport.error)")
            default:
                print("complete")
            }
        }
 }

Answer 2

So the problem is you forgot to add the audio track from you video file to the composition.

To solve this problem: (To avoid confusion, rename audioUrl to musicUrl here, and rename mutableCompositionAudioTrack to mutableCompositionAudioTracks as it’s an array.)

1. Add two audio tracks to mixComposition
2. Add music assetTrack and audio assetTrack from video to above audio tracks respectively. Both in the same time range.
3. Optional: if you need control the volume of each audio tracks, add an AVAudioMix to do it.

Here is the code with updated Swift 5, without the export part:

open func mergeVideoWithAudio(videoUrl: URL, musicUrl: URL){
        
        let mixComposition: AVMutableComposition = AVMutableComposition()
        var mutableCompositionVideoTracks: [AVMutableCompositionTrack] = []
        var mutableCompositionAudioTracks: [AVMutableCompositionTrack] = []
        let totalVideoCompositionInstruction : AVMutableVideoCompositionInstruction = AVMutableVideoCompositionInstruction()
        
        let aVideoAsset: AVAsset = AVAsset(url: videoUrl)
        let aMusicAsset: AVAsset = AVAsset(url: musicUrl)
        
        if let videoTrack = mixComposition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid),
           let audioTrack = mixComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid),
           let musicTrack = mixComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) {
            mutableCompositionVideoTracks.append(videoTrack)
            mutableCompositionAudioTracks.append(audioTrack)
            mutableCompositionAudioTracks.append(musicTrack)
        }
        
        let range = CMTimeRange(start: .zero, duration: aVideoAsset.duration)
        
        if let aVideoAssetTrack: AVAssetTrack = aVideoAsset.tracks(withMediaType: .video).first,
           let aAudioAssetTrack: AVAssetTrack = aVideoAsset.tracks(withMediaType: .audio).first,
           let aMusicAssetTrack: AVAssetTrack = aMusicAsset.tracks(withMediaType: .audio).first {
            do {
                try mutableCompositionVideoTracks.first?.insertTimeRange(range, of: aVideoAssetTrack, at: .zero)
                
                try mutableCompositionAudioTracks[0].insertTimeRange(range, of: aAudioAssetTrack, at: .zero)
                try mutableCompositionAudioTracks[1].insertTimeRange(range, of: aMusicAssetTrack, at: .zero)
                
            } catch{
                print(error)
            }
            
            totalVideoCompositionInstruction.timeRange = CMTimeRangeMake(start: CMTime.zero,duration: aVideoAssetTrack.timeRange.duration)
            
            let compositionV = mixComposition.tracks(withMediaType: AVMediaType.video).last
            if ((aVideoAssetTrack != nil) && (compositionV != nil)) {
                compositionV?.preferredTransform = (aVideoAssetTrack.preferredTransform)
            }
        }
        
        
        // export here
        ...
    }