Files

301 lines
18 KiB
Swift

// SPDX-License-Identifier: GPL-3.0-or-later
import Foundation
import AVFoundation
import CoreGraphics
import CoreVideo
import AudioToolbox
/// Only emulator pixels and generated stereo PCM enter this boundary. There is
/// deliberately no host-window, microphone, or screen-recording API here.
struct CoCoVideoFrame: Sendable {
var pixels: Data
var area: CGRect
var audio: Data // packed, interleaved Float32 stereo, 44,100 sample frames/sec
var sampleOffset: Int64
}
enum CoCoRecordingError: LocalizedError {
case message(String)
var errorDescription: String? { if case let .message(s) = self { return s }; return nil }
}
/// Serial, bounded writer. Working MOV fragments remain available after a
/// failure; only a verified MP4 completion permits their owner to clean up.
final class CoCoVideoEncoder: @unchecked Sendable {
static let width = 1280, height = 960
static let rate: Int32 = 44_100
let workingURL: URL
private let queue = DispatchQueue(label: "com.brianholman.vccplus.video-encoder", qos: .userInitiated)
private let writer: AVAssetWriter
private let video: AVAssetWriterInput
private let audio: AVAssetWriterInput
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
private var audioFormat: CMAudioFormatDescription?
private var images: [(Data, CGRect, Int64)] = []
private var sounds: [(Data, Int64)] = []
private var queuedSamples = 0
private var lastVideoTick: Int64 = -1
private var previousPicture: (Data, CGRect)?
private var lastAudioEnd: Int64 = 0
private var finalSample: Int64 = 0
private var finishing = false, draining = false, failed = false
private var finishReplies: [(Result<URL, Error>) -> Void] = []
private var finishResult: Result<URL, Error>?
private var closingWriter = false
private var failureError: Error?
private let ingressLock = NSLock()
private var accepting = true
private var retainedFrames = 0, retainedSamples = 0
var failure: (@Sendable (String) -> Void)?
/// Read-only AVFoundation lifecycle evidence; a finish reply promises this
/// terminal state, including when a failed movie is retained for recovery.
var writerIsTerminal: Bool { [.completed, .failed, .cancelled].contains(writer.status) }
init(workingURL: URL) throws {
self.workingURL = workingURL
writer = try AVAssetWriter(outputURL: workingURL, fileType: .mov)
writer.movieFragmentInterval = CMTime(seconds: 10, preferredTimescale: 600)
writer.initialMovieFragmentInterval = CMTime(seconds: 1, preferredTimescale: 600)
video = AVAssetWriterInput(mediaType: .video, outputSettings: [
AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: Self.width,
AVVideoHeightKey: Self.height, AVVideoCompressionPropertiesKey: [
AVVideoAverageBitRateKey: 4_000_000, AVVideoExpectedSourceFrameRateKey: 60,
AVVideoMaxKeyFrameIntervalKey: 60, AVVideoAllowFrameReorderingKey: false]])
audio = AVAssetWriterInput(mediaType: .audio, outputSettings: [
AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: 48_000,
AVNumberOfChannelsKey: 2, AVEncoderBitRateKey: 160_000])
video.expectsMediaDataInRealTime = true; audio.expectsMediaDataInRealTime = true
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: video,
sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey as String: Self.width, kCVPixelBufferHeightKey as String: Self.height,
kCVPixelBufferCGImageCompatibilityKey as String: true,
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true])
guard writer.canAdd(video), writer.canAdd(audio) else { throw CoCoRecordingError.message("The native video/audio encoder is unavailable.") }
writer.add(video); writer.add(audio)
var format = AudioStreamBasicDescription(mSampleRate: Double(Self.rate), mFormatID: kAudioFormatLinearPCM,
mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked,
mBytesPerPacket: 8, mFramesPerPacket: 1, mBytesPerFrame: 8, mChannelsPerFrame: 2,
mBitsPerChannel: 32, mReserved: 0)
guard CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &format,
layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil,
extensions: nil, formatDescriptionOut: &audioFormat) == noErr,
writer.startWriting() else { throw writer.error ?? CoCoRecordingError.message("The recording could not start.") }
writer.startSession(atSourceTime: .zero)
}
func append(_ frames: [CoCoVideoFrame]) {
let samples = frames.reduce(0) { $0 + $1.audio.count / 8 }
ingressLock.lock()
guard accepting else { ingressLock.unlock(); return }
guard retainedFrames + frames.count <= 12, retainedSamples + samples <= Int(Self.rate) * 2 else {
accepting = false; ingressLock.unlock()
queue.async { [self] in fail(CoCoRecordingError.message("The recorder reached its bounded input limit. The working movie has been retained.")) }
return
}
retainedFrames += frames.count; retainedSamples += samples
ingressLock.unlock()
queue.async { [self] in
guard !failed, !finishing else { return }
do {
for frame in frames {
guard frame.pixels.count == 640 * 480 * 4, frame.audio.count % 8 == 0,
frame.sampleOffset >= lastAudioEnd else {
throw CoCoRecordingError.message("The recording input lost its frame/audio ordering.")
}
let gap = frame.sampleOffset - lastAudioEnd
guard gap <= Int64(Self.rate) * 2 else { throw CoCoRecordingError.message("The recording input was interrupted. The working movie has been retained.") }
if gap > 0 {
ingressLock.lock()
let allowed = retainedSamples + Int(gap) <= Int(Self.rate) * 2
if allowed { retainedSamples += Int(gap) }
ingressLock.unlock()
guard allowed else { throw CoCoRecordingError.message("The recorder reached its bounded audio limit.") }
sounds.append((Data(count: Int(gap) * 8), lastAudioEnd)); queuedSamples += Int(gap)
}
if !frame.audio.isEmpty { sounds.append((frame.audio, frame.sampleOffset)); queuedSamples += frame.audio.count / 8 }
lastAudioEnd = frame.sampleOffset + Int64(frame.audio.count / 8)
finalSample = max(finalSample, lastAudioEnd)
let tick = frame.sampleOffset * 60 / Int64(Self.rate)
if tick > lastVideoTick {
let duplicates = Int(tick - lastVideoTick - 1)
if duplicates > 0 {
ingressLock.lock()
let allowed = retainedFrames + duplicates <= 12
if allowed { retainedFrames += duplicates }
ingressLock.unlock()
guard allowed else { throw CoCoRecordingError.message("The recording frame clock was interrupted. The working movie is retained.") }
let held = previousPicture ?? (frame.pixels, frame.area)
for t in (lastVideoTick + 1)..<tick { images.append((held.0, held.1, t)) }
}
images.append((frame.pixels, frame.area, tick)); lastVideoTick = tick
previousPicture = (frame.pixels, frame.area)
}
else { release(frames: 1, samples: 0) }
guard images.count <= 12, queuedSamples <= Int(Self.rate) * 2 else {
throw CoCoRecordingError.message("The video encoder could not keep up. The working movie has been retained.")
}
}
drain()
} catch { fail(error) }
}
}
func finish(_ completion: @escaping (Result<URL, Error>) -> Void) {
ingressLock.lock(); accepting = false; ingressLock.unlock()
queue.async { [self] in
if let finishResult { completion(finishResult); return }
finishReplies.append(completion)
guard !finishing, !failed else { return }
finishing = true; drain()
}
}
/// A disposable producer advances faster than wall time. Its own worker may
/// await this acknowledgement rather than overflowing the unchanged bounded
/// ingress queue. The application recorder continues using nonblocking append.
func whenDrained(_ completion: @escaping @Sendable (Result<Void, Error>) -> Void) {
let deadline = ProcessInfo.processInfo.systemUptime + 10
func check() {
queue.async { [self] in
if failed { completion(.failure(writer.error ?? CoCoRecordingError.message("The video writer stopped."))); return }
ingressLock.lock()
let empty = retainedFrames == 0 && retainedSamples == 0
ingressLock.unlock()
if empty { completion(.success(())) }
else if ProcessInfo.processInfo.systemUptime >= deadline {
let error = CoCoRecordingError.message("The video writer did not drain within its deadline.")
fail(error); completion(.failure(error))
} else { queue.asyncAfter(deadline: .now() + .milliseconds(5), execute: check) }
}
}
check()
}
private func drain() {
guard !failed else { return }
do {
while !images.isEmpty && video.isReadyForMoreMediaData {
let image = images.removeFirst()
let pixel = try Self.pixelBuffer(image.0, area: image.1, pool: adaptor.pixelBufferPool)
guard adaptor.append(pixel, withPresentationTime: CMTime(value: image.2, timescale: 60)) else {
throw writer.error ?? CoCoRecordingError.message("A video frame could not be written.")
}
release(frames: 1, samples: 0)
}
while !sounds.isEmpty && audio.isReadyForMoreMediaData {
let sound = sounds.removeFirst(); queuedSamples -= sound.0.count / 8
let sample = try audioSample(sound.0, offset: sound.1)
guard audio.append(sample) else { throw writer.error ?? CoCoRecordingError.message("CoCo sound could not be written.") }
release(frames: 0, samples: sound.0.count / 8)
}
if writer.status == .failed { throw writer.error ?? CoCoRecordingError.message("The movie writer stopped.") }
if images.isEmpty && sounds.isEmpty {
if finishing, !closingWriter {
closingWriter = true
writer.endSession(atSourceTime: CMTime(value: max(finalSample, 735), timescale: Self.rate))
video.markAsFinished(); audio.markAsFinished()
writer.finishWriting { [self] in
queue.async { [self] in
if let failureError { finishAll(.failure(failureError)) }
else if writer.status == .completed { finishAll(.success(workingURL)) }
else { finishAll(.failure(writer.error ?? CoCoRecordingError.message("The working movie could not be finalized."))) }
}
}
}
} else if !draining {
draining = true
queue.asyncAfter(deadline: .now() + .milliseconds(5)) { [self] in draining = false; drain() }
}
} catch { fail(error) }
}
private func fail(_ error: Error) {
guard !failed else { return }; failed = true
failureError = error
ingressLock.lock(); accepting = false; retainedFrames = 0; retainedSamples = 0; ingressLock.unlock()
images.removeAll(); sounds.removeAll(); queuedSamples = 0
// Finishing what the framework can retain is preferable to cancelling
// and deleting fragments. Every Stop waits for that closure, including
// Stop arriving after the failure callback or during another Stop.
if closingWriter {
// The existing AVFoundation callback owns the terminal result.
} else if writer.status == .writing {
closingWriter = true
video.markAsFinished(); audio.markAsFinished()
writer.finishWriting { [self] in queue.async { [self] in finishAll(.failure(error)) } }
} else { finishAll(.failure(error)) }
failure?(error.localizedDescription)
}
private func finishAll(_ result: Result<URL, Error>) {
guard finishResult == nil else { return }
finishResult = result
let replies = finishReplies; finishReplies.removeAll()
for reply in replies { reply(result) }
}
private func release(frames: Int, samples: Int) {
ingressLock.lock(); retainedFrames -= frames; retainedSamples -= samples; ingressLock.unlock()
}
private func audioSample(_ data: Data, offset: Int64) throws -> CMSampleBuffer {
var block: CMBlockBuffer?
guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil,
blockLength: data.count, blockAllocator: kCFAllocatorDefault, customBlockSource: nil,
offsetToData: 0, dataLength: data.count, flags: 0, blockBufferOut: &block) == kCMBlockBufferNoErr,
let block else { throw CoCoRecordingError.message("Audio buffer allocation failed.") }
let copied = data.withUnsafeBytes { CMBlockBufferReplaceDataBytes(with: $0.baseAddress!, blockBuffer: block, offsetIntoDestination: 0, dataLength: data.count) }
var time = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: Self.rate),
presentationTimeStamp: CMTime(value: offset, timescale: Self.rate), decodeTimeStamp: .invalid)
var sample: CMSampleBuffer?
guard copied == noErr, CMSampleBufferCreateReady(allocator: kCFAllocatorDefault,
dataBuffer: block, formatDescription: audioFormat, sampleCount: data.count / 8,
sampleTimingEntryCount: 1, sampleTimingArray: &time, sampleSizeEntryCount: 0,
sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, let sample else {
throw CoCoRecordingError.message("The CoCo audio sample could not be prepared.")
}
return sample
}
static func pixelBuffer(_ bytes: Data, area: CGRect, pool: CVPixelBufferPool?) throws -> CVPixelBuffer {
guard bytes.count == 640 * 480 * 4 else { throw CoCoRecordingError.message("No complete CoCo picture is available.") }
guard [area.minX, area.minY, area.width, area.height].allSatisfy(\.isFinite) else {
throw CoCoRecordingError.message("The recording picture dimensions are invalid.")
}
let bounds = CGRect(x: 0, y: 0, width: 640, height: 480)
let rect = area.intersection(bounds).integral
guard !rect.isEmpty, !rect.isNull, let pool else { throw CoCoRecordingError.message("The recording picture dimensions are invalid.") }
var buffer: CVPixelBuffer?
guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess, let buffer else { throw CoCoRecordingError.message("A video pixel buffer could not be allocated.") }
CVPixelBufferLockBaseAddress(buffer, []); defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
let color = CGColorSpace(name: CGColorSpace.sRGB)!
guard let source = CGDataProvider(data: bytes as CFData),
let image = CGImage(width: 640, height: 480, bitsPerComponent: 8, bitsPerPixel: 32,
bytesPerRow: 640 * 4, space: color,
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue),
provider: source, decode: nil, shouldInterpolate: false, intent: .defaultIntent)?.cropping(to: rect),
let context = CGContext(data: CVPixelBufferGetBaseAddress(buffer), width: width, height: height,
bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: color,
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else {
throw CoCoRecordingError.message("The CoCo picture could not be rendered for video.")
}
context.setFillColor(CGColor(gray: 0, alpha: 1)); context.fill(CGRect(x: 0, y: 0, width: width, height: height))
let scale = min(CGFloat(width) / rect.width, CGFloat(height) / rect.height)
let size = CGSize(width: rect.width * scale, height: rect.height * scale)
context.interpolationQuality = .none
context.draw(image, in: CGRect(x: (CGFloat(width)-size.width)/2, y: (CGFloat(height)-size.height)/2, width: size.width, height: size.height))
return buffer
}
static func exportMP4(from source: URL, to destination: URL) async throws {
guard !FileManager.default.fileExists(atPath: destination.path),
let export = AVAssetExportSession(asset: AVURLAsset(url: source), presetName: AVAssetExportPresetPassthrough),
export.supportedFileTypes.contains(.mp4) else { throw CoCoRecordingError.message("The retained movie cannot be converted to this MP4 destination.") }
export.outputURL = destination; export.outputFileType = .mp4
await export.export()
guard export.status == .completed else { throw export.error ?? CoCoRecordingError.message("MP4 completion failed; the working movie has been retained.") }
let asset = AVURLAsset(url: destination)
guard try await !asset.loadTracks(withMediaType: .video).isEmpty,
try await !asset.loadTracks(withMediaType: .audio).isEmpty,
try await asset.load(.duration).seconds > 0 else { throw CoCoRecordingError.message("The completed movie did not contain playable video and audio.") }
}
}