Hi,
I'm trying to insert CMSampleBuffers into an AVAssetWriterInput that has been configured with expectsMediaDataInRealTime = false with pauses. That is, I insert fixed-length audio at specific (increasing and non-overlapping) time points with large gaps in between. E.g., 5 seconds of audio at t=3.0, 5 seconds of audio at t=12.0, etc.
The first audio sample plays at t=3 in the final output video as expected. But then all the other samples are bunched up immediately after it instead of being scheduled at the correct time. Below is my code.
I'm just loading the asset and then readjusting its timestamps to be correct in the absolute timeline. Why do they not get scheduled correctly when the timestamps and durations are definitely correct and non-overlapping?
func addFrame(_ pixelBuffer: CVPixelBuffer) {
guard CGSize(width: pixelBuffer.width, height: pixelBuffer.height) == outputSize else { return }
let frameTime = CMTimeMake(value: frameCount, timescale: frameRate)
if videoInput?.isReadyForMoreMediaData == true {
pixelBufferAdaptor?.append(pixelBuffer, withPresentationTime: frameTime)
frameCount += 1
currentTime = frameTime
}
}
func addMP3AudioClip(_ audioData: Data) async throws {
let tempURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mp3")
defer {
try? FileManager.default.removeItem(at: tempURL)
}
try audioData.write(to: tempURL)
let asset = AVAsset(url: tempURL)
let duration = try await asset.load(.duration)
let audioTrack = try await asset.loadTracks(withMediaType: .audio).first!
let audioReader = try AVAssetReader(asset: asset)
let outputSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatLinearPCM,
AVSampleRateKey: 44100,
AVNumberOfChannelsKey: 2,
AVLinearPCMBitDepthKey: 16,
AVLinearPCMIsFloatKey: false,
AVLinearPCMIsBigEndianKey: false,
AVLinearPCMIsNonInterleaved: false
]
let audioReaderOutput = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: outputSettings)
audioReader.add(audioReaderOutput)
guard audioReader.startReading() else {
throw NSError(domain: "AudioReaderError", code: 0, userInfo: [NSLocalizedDescriptionKey: "Failed to start reading audio"])
}
let baseInsertionTime = currentTime.convertScale(duration.timescale, method: .default) // Capture the current video time when the method is called
print("Adding audio clip at \(baseInsertionTime.seconds) seconds, duration: \(duration.seconds) seconds")
var audioTime = CMTime.zero
var totalDuration: Double = 0
while let sampleBuffer = audioReaderOutput.copyNextSampleBuffer() {
let bufferDuration = CMSampleBufferGetDuration(sampleBuffer)
let adjustedBuffer = adjustTimeStamp(of: sampleBuffer, by: baseInsertionTime)
while !audioInput!.isReadyForMoreMediaData {
try await Task.sleep(nanoseconds: 100_000_000) // 0.1 second
}
audioInput!.append(adjustedBuffer)
print(" Adjusted time: \(adjustedBuffer.presentationTimeStamp.seconds)")
audioTime = CMTimeAdd(audioTime, bufferDuration)
totalDuration += bufferDuration.seconds
}
print("Finished adding audio clip. Last sample at: \(CMTimeAdd(baseInsertionTime, audioTime).seconds) seconds")
print(" totalDuration=\(totalDuration)")
}
private func adjustTimeStamp(of sampleBuffer: CMSampleBuffer, by timeOffset: CMTime) -> CMSampleBuffer {
var count: CMItemCount = 0
CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, entryCount: 0, arrayToFill: nil, entriesNeededOut: &count)
var timingInfo = [CMSampleTimingInfo](repeating: CMSampleTimingInfo(), count: count)
CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, entryCount: count, arrayToFill: &timingInfo, entriesNeededOut: nil)
for i in 0..<count {
timingInfo[i].presentationTimeStamp = CMTimeAdd(timingInfo[i].presentationTimeStamp, timeOffset)
if timingInfo[i].decodeTimeStamp != .invalid {
timingInfo[i].decodeTimeStamp = CMTimeAdd(timingInfo[i].decodeTimeStamp, timeOffset)
} else {
timingInfo[i].decodeTimeStamp = timingInfo[i].presentationTimeStamp
}
}
var adjustedBuffer: CMSampleBuffer?
CMSampleBufferCreateCopyWithNewTiming(allocator: nil, sampleBuffer: sampleBuffer, sampleTimingEntryCount: count, sampleTimingArray: &timingInfo, sampleBufferOut: &adjustedBuffer)
return adjustedBuffer!
}
