301 lines
18 KiB
Swift
301 lines
18 KiB
Swift
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
import Foundation
|
|
import AVFoundation
|
|
import CoreGraphics
|
|
import CoreVideo
|
|
import AudioToolbox
|
|
|
|
/// Only emulator pixels and generated stereo PCM enter this boundary. There is
|
|
/// deliberately no host-window, microphone, or screen-recording API here.
|
|
struct CoCoVideoFrame: Sendable {
|
|
var pixels: Data
|
|
var area: CGRect
|
|
var audio: Data // packed, interleaved Float32 stereo, 44,100 sample frames/sec
|
|
var sampleOffset: Int64
|
|
}
|
|
|
|
enum CoCoRecordingError: LocalizedError {
|
|
case message(String)
|
|
var errorDescription: String? { if case let .message(s) = self { return s }; return nil }
|
|
}
|
|
|
|
/// Serial, bounded writer. Working MOV fragments remain available after a
|
|
/// failure; only a verified MP4 completion permits their owner to clean up.
|
|
final class CoCoVideoEncoder: @unchecked Sendable {
|
|
static let width = 1280, height = 960
|
|
static let rate: Int32 = 44_100
|
|
let workingURL: URL
|
|
private let queue = DispatchQueue(label: "com.brianholman.vccplus.video-encoder", qos: .userInitiated)
|
|
private let writer: AVAssetWriter
|
|
private let video: AVAssetWriterInput
|
|
private let audio: AVAssetWriterInput
|
|
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
|
|
private var audioFormat: CMAudioFormatDescription?
|
|
private var images: [(Data, CGRect, Int64)] = []
|
|
private var sounds: [(Data, Int64)] = []
|
|
private var queuedSamples = 0
|
|
private var lastVideoTick: Int64 = -1
|
|
private var previousPicture: (Data, CGRect)?
|
|
private var lastAudioEnd: Int64 = 0
|
|
private var finalSample: Int64 = 0
|
|
private var finishing = false, draining = false, failed = false
|
|
private var finishReplies: [(Result<URL, Error>) -> Void] = []
|
|
private var finishResult: Result<URL, Error>?
|
|
private var closingWriter = false
|
|
private var failureError: Error?
|
|
private let ingressLock = NSLock()
|
|
private var accepting = true
|
|
private var retainedFrames = 0, retainedSamples = 0
|
|
var failure: (@Sendable (String) -> Void)?
|
|
/// Read-only AVFoundation lifecycle evidence; a finish reply promises this
|
|
/// terminal state, including when a failed movie is retained for recovery.
|
|
var writerIsTerminal: Bool { [.completed, .failed, .cancelled].contains(writer.status) }
|
|
|
|
init(workingURL: URL) throws {
|
|
self.workingURL = workingURL
|
|
writer = try AVAssetWriter(outputURL: workingURL, fileType: .mov)
|
|
writer.movieFragmentInterval = CMTime(seconds: 10, preferredTimescale: 600)
|
|
writer.initialMovieFragmentInterval = CMTime(seconds: 1, preferredTimescale: 600)
|
|
video = AVAssetWriterInput(mediaType: .video, outputSettings: [
|
|
AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: Self.width,
|
|
AVVideoHeightKey: Self.height, AVVideoCompressionPropertiesKey: [
|
|
AVVideoAverageBitRateKey: 4_000_000, AVVideoExpectedSourceFrameRateKey: 60,
|
|
AVVideoMaxKeyFrameIntervalKey: 60, AVVideoAllowFrameReorderingKey: false]])
|
|
audio = AVAssetWriterInput(mediaType: .audio, outputSettings: [
|
|
AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: 48_000,
|
|
AVNumberOfChannelsKey: 2, AVEncoderBitRateKey: 160_000])
|
|
video.expectsMediaDataInRealTime = true; audio.expectsMediaDataInRealTime = true
|
|
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: video,
|
|
sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
|
kCVPixelBufferWidthKey as String: Self.width, kCVPixelBufferHeightKey as String: Self.height,
|
|
kCVPixelBufferCGImageCompatibilityKey as String: true,
|
|
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true])
|
|
guard writer.canAdd(video), writer.canAdd(audio) else { throw CoCoRecordingError.message("The native video/audio encoder is unavailable.") }
|
|
writer.add(video); writer.add(audio)
|
|
var format = AudioStreamBasicDescription(mSampleRate: Double(Self.rate), mFormatID: kAudioFormatLinearPCM,
|
|
mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked,
|
|
mBytesPerPacket: 8, mFramesPerPacket: 1, mBytesPerFrame: 8, mChannelsPerFrame: 2,
|
|
mBitsPerChannel: 32, mReserved: 0)
|
|
guard CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &format,
|
|
layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil,
|
|
extensions: nil, formatDescriptionOut: &audioFormat) == noErr,
|
|
writer.startWriting() else { throw writer.error ?? CoCoRecordingError.message("The recording could not start.") }
|
|
writer.startSession(atSourceTime: .zero)
|
|
}
|
|
|
|
func append(_ frames: [CoCoVideoFrame]) {
|
|
let samples = frames.reduce(0) { $0 + $1.audio.count / 8 }
|
|
ingressLock.lock()
|
|
guard accepting else { ingressLock.unlock(); return }
|
|
guard retainedFrames + frames.count <= 12, retainedSamples + samples <= Int(Self.rate) * 2 else {
|
|
accepting = false; ingressLock.unlock()
|
|
queue.async { [self] in fail(CoCoRecordingError.message("The recorder reached its bounded input limit. The working movie has been retained.")) }
|
|
return
|
|
}
|
|
retainedFrames += frames.count; retainedSamples += samples
|
|
ingressLock.unlock()
|
|
queue.async { [self] in
|
|
guard !failed, !finishing else { return }
|
|
do {
|
|
for frame in frames {
|
|
guard frame.pixels.count == 640 * 480 * 4, frame.audio.count % 8 == 0,
|
|
frame.sampleOffset >= lastAudioEnd else {
|
|
throw CoCoRecordingError.message("The recording input lost its frame/audio ordering.")
|
|
}
|
|
let gap = frame.sampleOffset - lastAudioEnd
|
|
guard gap <= Int64(Self.rate) * 2 else { throw CoCoRecordingError.message("The recording input was interrupted. The working movie has been retained.") }
|
|
if gap > 0 {
|
|
ingressLock.lock()
|
|
let allowed = retainedSamples + Int(gap) <= Int(Self.rate) * 2
|
|
if allowed { retainedSamples += Int(gap) }
|
|
ingressLock.unlock()
|
|
guard allowed else { throw CoCoRecordingError.message("The recorder reached its bounded audio limit.") }
|
|
sounds.append((Data(count: Int(gap) * 8), lastAudioEnd)); queuedSamples += Int(gap)
|
|
}
|
|
if !frame.audio.isEmpty { sounds.append((frame.audio, frame.sampleOffset)); queuedSamples += frame.audio.count / 8 }
|
|
lastAudioEnd = frame.sampleOffset + Int64(frame.audio.count / 8)
|
|
finalSample = max(finalSample, lastAudioEnd)
|
|
let tick = frame.sampleOffset * 60 / Int64(Self.rate)
|
|
if tick > lastVideoTick {
|
|
let duplicates = Int(tick - lastVideoTick - 1)
|
|
if duplicates > 0 {
|
|
ingressLock.lock()
|
|
let allowed = retainedFrames + duplicates <= 12
|
|
if allowed { retainedFrames += duplicates }
|
|
ingressLock.unlock()
|
|
guard allowed else { throw CoCoRecordingError.message("The recording frame clock was interrupted. The working movie is retained.") }
|
|
let held = previousPicture ?? (frame.pixels, frame.area)
|
|
for t in (lastVideoTick + 1)..<tick { images.append((held.0, held.1, t)) }
|
|
}
|
|
images.append((frame.pixels, frame.area, tick)); lastVideoTick = tick
|
|
previousPicture = (frame.pixels, frame.area)
|
|
}
|
|
else { release(frames: 1, samples: 0) }
|
|
guard images.count <= 12, queuedSamples <= Int(Self.rate) * 2 else {
|
|
throw CoCoRecordingError.message("The video encoder could not keep up. The working movie has been retained.")
|
|
}
|
|
}
|
|
drain()
|
|
} catch { fail(error) }
|
|
}
|
|
}
|
|
|
|
func finish(_ completion: @escaping (Result<URL, Error>) -> Void) {
|
|
ingressLock.lock(); accepting = false; ingressLock.unlock()
|
|
queue.async { [self] in
|
|
if let finishResult { completion(finishResult); return }
|
|
finishReplies.append(completion)
|
|
guard !finishing, !failed else { return }
|
|
finishing = true; drain()
|
|
}
|
|
}
|
|
|
|
/// A disposable producer advances faster than wall time. Its own worker may
|
|
/// await this acknowledgement rather than overflowing the unchanged bounded
|
|
/// ingress queue. The application recorder continues using nonblocking append.
|
|
func whenDrained(_ completion: @escaping @Sendable (Result<Void, Error>) -> Void) {
|
|
let deadline = ProcessInfo.processInfo.systemUptime + 10
|
|
func check() {
|
|
queue.async { [self] in
|
|
if failed { completion(.failure(writer.error ?? CoCoRecordingError.message("The video writer stopped."))); return }
|
|
ingressLock.lock()
|
|
let empty = retainedFrames == 0 && retainedSamples == 0
|
|
ingressLock.unlock()
|
|
if empty { completion(.success(())) }
|
|
else if ProcessInfo.processInfo.systemUptime >= deadline {
|
|
let error = CoCoRecordingError.message("The video writer did not drain within its deadline.")
|
|
fail(error); completion(.failure(error))
|
|
} else { queue.asyncAfter(deadline: .now() + .milliseconds(5), execute: check) }
|
|
}
|
|
}
|
|
check()
|
|
}
|
|
|
|
private func drain() {
|
|
guard !failed else { return }
|
|
do {
|
|
while !images.isEmpty && video.isReadyForMoreMediaData {
|
|
let image = images.removeFirst()
|
|
let pixel = try Self.pixelBuffer(image.0, area: image.1, pool: adaptor.pixelBufferPool)
|
|
guard adaptor.append(pixel, withPresentationTime: CMTime(value: image.2, timescale: 60)) else {
|
|
throw writer.error ?? CoCoRecordingError.message("A video frame could not be written.")
|
|
}
|
|
release(frames: 1, samples: 0)
|
|
}
|
|
while !sounds.isEmpty && audio.isReadyForMoreMediaData {
|
|
let sound = sounds.removeFirst(); queuedSamples -= sound.0.count / 8
|
|
let sample = try audioSample(sound.0, offset: sound.1)
|
|
guard audio.append(sample) else { throw writer.error ?? CoCoRecordingError.message("CoCo sound could not be written.") }
|
|
release(frames: 0, samples: sound.0.count / 8)
|
|
}
|
|
if writer.status == .failed { throw writer.error ?? CoCoRecordingError.message("The movie writer stopped.") }
|
|
if images.isEmpty && sounds.isEmpty {
|
|
if finishing, !closingWriter {
|
|
closingWriter = true
|
|
writer.endSession(atSourceTime: CMTime(value: max(finalSample, 735), timescale: Self.rate))
|
|
video.markAsFinished(); audio.markAsFinished()
|
|
writer.finishWriting { [self] in
|
|
queue.async { [self] in
|
|
if let failureError { finishAll(.failure(failureError)) }
|
|
else if writer.status == .completed { finishAll(.success(workingURL)) }
|
|
else { finishAll(.failure(writer.error ?? CoCoRecordingError.message("The working movie could not be finalized."))) }
|
|
}
|
|
}
|
|
}
|
|
} else if !draining {
|
|
draining = true
|
|
queue.asyncAfter(deadline: .now() + .milliseconds(5)) { [self] in draining = false; drain() }
|
|
}
|
|
} catch { fail(error) }
|
|
}
|
|
|
|
private func fail(_ error: Error) {
|
|
guard !failed else { return }; failed = true
|
|
failureError = error
|
|
ingressLock.lock(); accepting = false; retainedFrames = 0; retainedSamples = 0; ingressLock.unlock()
|
|
images.removeAll(); sounds.removeAll(); queuedSamples = 0
|
|
// Finishing what the framework can retain is preferable to cancelling
|
|
// and deleting fragments. Every Stop waits for that closure, including
|
|
// Stop arriving after the failure callback or during another Stop.
|
|
if closingWriter {
|
|
// The existing AVFoundation callback owns the terminal result.
|
|
} else if writer.status == .writing {
|
|
closingWriter = true
|
|
video.markAsFinished(); audio.markAsFinished()
|
|
writer.finishWriting { [self] in queue.async { [self] in finishAll(.failure(error)) } }
|
|
} else { finishAll(.failure(error)) }
|
|
failure?(error.localizedDescription)
|
|
}
|
|
private func finishAll(_ result: Result<URL, Error>) {
|
|
guard finishResult == nil else { return }
|
|
finishResult = result
|
|
let replies = finishReplies; finishReplies.removeAll()
|
|
for reply in replies { reply(result) }
|
|
}
|
|
private func release(frames: Int, samples: Int) {
|
|
ingressLock.lock(); retainedFrames -= frames; retainedSamples -= samples; ingressLock.unlock()
|
|
}
|
|
|
|
private func audioSample(_ data: Data, offset: Int64) throws -> CMSampleBuffer {
|
|
var block: CMBlockBuffer?
|
|
guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil,
|
|
blockLength: data.count, blockAllocator: kCFAllocatorDefault, customBlockSource: nil,
|
|
offsetToData: 0, dataLength: data.count, flags: 0, blockBufferOut: &block) == kCMBlockBufferNoErr,
|
|
let block else { throw CoCoRecordingError.message("Audio buffer allocation failed.") }
|
|
let copied = data.withUnsafeBytes { CMBlockBufferReplaceDataBytes(with: $0.baseAddress!, blockBuffer: block, offsetIntoDestination: 0, dataLength: data.count) }
|
|
var time = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: Self.rate),
|
|
presentationTimeStamp: CMTime(value: offset, timescale: Self.rate), decodeTimeStamp: .invalid)
|
|
var sample: CMSampleBuffer?
|
|
guard copied == noErr, CMSampleBufferCreateReady(allocator: kCFAllocatorDefault,
|
|
dataBuffer: block, formatDescription: audioFormat, sampleCount: data.count / 8,
|
|
sampleTimingEntryCount: 1, sampleTimingArray: &time, sampleSizeEntryCount: 0,
|
|
sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, let sample else {
|
|
throw CoCoRecordingError.message("The CoCo audio sample could not be prepared.")
|
|
}
|
|
return sample
|
|
}
|
|
|
|
static func pixelBuffer(_ bytes: Data, area: CGRect, pool: CVPixelBufferPool?) throws -> CVPixelBuffer {
|
|
guard bytes.count == 640 * 480 * 4 else { throw CoCoRecordingError.message("No complete CoCo picture is available.") }
|
|
guard [area.minX, area.minY, area.width, area.height].allSatisfy(\.isFinite) else {
|
|
throw CoCoRecordingError.message("The recording picture dimensions are invalid.")
|
|
}
|
|
let bounds = CGRect(x: 0, y: 0, width: 640, height: 480)
|
|
let rect = area.intersection(bounds).integral
|
|
guard !rect.isEmpty, !rect.isNull, let pool else { throw CoCoRecordingError.message("The recording picture dimensions are invalid.") }
|
|
var buffer: CVPixelBuffer?
|
|
guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess, let buffer else { throw CoCoRecordingError.message("A video pixel buffer could not be allocated.") }
|
|
CVPixelBufferLockBaseAddress(buffer, []); defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
|
|
let color = CGColorSpace(name: CGColorSpace.sRGB)!
|
|
guard let source = CGDataProvider(data: bytes as CFData),
|
|
let image = CGImage(width: 640, height: 480, bitsPerComponent: 8, bitsPerPixel: 32,
|
|
bytesPerRow: 640 * 4, space: color,
|
|
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue),
|
|
provider: source, decode: nil, shouldInterpolate: false, intent: .defaultIntent)?.cropping(to: rect),
|
|
let context = CGContext(data: CVPixelBufferGetBaseAddress(buffer), width: width, height: height,
|
|
bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: color,
|
|
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else {
|
|
throw CoCoRecordingError.message("The CoCo picture could not be rendered for video.")
|
|
}
|
|
context.setFillColor(CGColor(gray: 0, alpha: 1)); context.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
|
let scale = min(CGFloat(width) / rect.width, CGFloat(height) / rect.height)
|
|
let size = CGSize(width: rect.width * scale, height: rect.height * scale)
|
|
context.interpolationQuality = .none
|
|
context.draw(image, in: CGRect(x: (CGFloat(width)-size.width)/2, y: (CGFloat(height)-size.height)/2, width: size.width, height: size.height))
|
|
return buffer
|
|
}
|
|
|
|
static func exportMP4(from source: URL, to destination: URL) async throws {
|
|
guard !FileManager.default.fileExists(atPath: destination.path),
|
|
let export = AVAssetExportSession(asset: AVURLAsset(url: source), presetName: AVAssetExportPresetPassthrough),
|
|
export.supportedFileTypes.contains(.mp4) else { throw CoCoRecordingError.message("The retained movie cannot be converted to this MP4 destination.") }
|
|
export.outputURL = destination; export.outputFileType = .mp4
|
|
await export.export()
|
|
guard export.status == .completed else { throw export.error ?? CoCoRecordingError.message("MP4 completion failed; the working movie has been retained.") }
|
|
let asset = AVURLAsset(url: destination)
|
|
guard try await !asset.loadTracks(withMediaType: .video).isEmpty,
|
|
try await !asset.loadTracks(withMediaType: .audio).isEmpty,
|
|
try await asset.load(.duration).seconds > 0 else { throw CoCoRecordingError.message("The completed movie did not contain playable video and audio.") }
|
|
}
|
|
}
|