// SPDX-License-Identifier: GPL-3.0-or-later import Foundation import AVFoundation import CoreGraphics import CoreVideo import AudioToolbox /// Only emulator pixels and generated stereo PCM enter this boundary. There is /// deliberately no host-window, microphone, or screen-recording API here. struct CoCoVideoFrame: Sendable { var pixels: Data var area: CGRect var audio: Data // packed, interleaved Float32 stereo, 44,100 sample frames/sec var sampleOffset: Int64 } enum CoCoRecordingError: LocalizedError { case message(String) var errorDescription: String? { if case let .message(s) = self { return s }; return nil } } /// Serial, bounded writer. Working MOV fragments remain available after a /// failure; only a verified MP4 completion permits their owner to clean up. final class CoCoVideoEncoder: @unchecked Sendable { static let width = 1280, height = 960 static let rate: Int32 = 44_100 let workingURL: URL private let queue = DispatchQueue(label: "com.brianholman.vccplus.video-encoder", qos: .userInitiated) private let writer: AVAssetWriter private let video: AVAssetWriterInput private let audio: AVAssetWriterInput private let adaptor: AVAssetWriterInputPixelBufferAdaptor private var audioFormat: CMAudioFormatDescription? private var images: [(Data, CGRect, Int64)] = [] private var sounds: [(Data, Int64)] = [] private var queuedSamples = 0 private var lastVideoTick: Int64 = -1 private var previousPicture: (Data, CGRect)? private var lastAudioEnd: Int64 = 0 private var finalSample: Int64 = 0 private var finishing = false, draining = false, failed = false private var finishReplies: [(Result) -> Void] = [] private var finishResult: Result? private var closingWriter = false private var failureError: Error? private let ingressLock = NSLock() private var accepting = true private var retainedFrames = 0, retainedSamples = 0 var failure: (@Sendable (String) -> Void)? /// Read-only AVFoundation lifecycle evidence; a finish reply promises this /// terminal state, including when a failed movie is retained for recovery. var writerIsTerminal: Bool { [.completed, .failed, .cancelled].contains(writer.status) } init(workingURL: URL) throws { self.workingURL = workingURL writer = try AVAssetWriter(outputURL: workingURL, fileType: .mov) writer.movieFragmentInterval = CMTime(seconds: 10, preferredTimescale: 600) writer.initialMovieFragmentInterval = CMTime(seconds: 1, preferredTimescale: 600) video = AVAssetWriterInput(mediaType: .video, outputSettings: [ AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: Self.width, AVVideoHeightKey: Self.height, AVVideoCompressionPropertiesKey: [ AVVideoAverageBitRateKey: 4_000_000, AVVideoExpectedSourceFrameRateKey: 60, AVVideoMaxKeyFrameIntervalKey: 60, AVVideoAllowFrameReorderingKey: false]]) audio = AVAssetWriterInput(mediaType: .audio, outputSettings: [ AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: 48_000, AVNumberOfChannelsKey: 2, AVEncoderBitRateKey: 160_000]) video.expectsMediaDataInRealTime = true; audio.expectsMediaDataInRealTime = true adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: video, sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, kCVPixelBufferWidthKey as String: Self.width, kCVPixelBufferHeightKey as String: Self.height, kCVPixelBufferCGImageCompatibilityKey as String: true, kCVPixelBufferCGBitmapContextCompatibilityKey as String: true]) guard writer.canAdd(video), writer.canAdd(audio) else { throw CoCoRecordingError.message("The native video/audio encoder is unavailable.") } writer.add(video); writer.add(audio) var format = AudioStreamBasicDescription(mSampleRate: Double(Self.rate), mFormatID: kAudioFormatLinearPCM, mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked, mBytesPerPacket: 8, mFramesPerPacket: 1, mBytesPerFrame: 8, mChannelsPerFrame: 2, mBitsPerChannel: 32, mReserved: 0) guard CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &format, layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil, extensions: nil, formatDescriptionOut: &audioFormat) == noErr, writer.startWriting() else { throw writer.error ?? CoCoRecordingError.message("The recording could not start.") } writer.startSession(atSourceTime: .zero) } func append(_ frames: [CoCoVideoFrame]) { let samples = frames.reduce(0) { $0 + $1.audio.count / 8 } ingressLock.lock() guard accepting else { ingressLock.unlock(); return } guard retainedFrames + frames.count <= 12, retainedSamples + samples <= Int(Self.rate) * 2 else { accepting = false; ingressLock.unlock() queue.async { [self] in fail(CoCoRecordingError.message("The recorder reached its bounded input limit. The working movie has been retained.")) } return } retainedFrames += frames.count; retainedSamples += samples ingressLock.unlock() queue.async { [self] in guard !failed, !finishing else { return } do { for frame in frames { guard frame.pixels.count == 640 * 480 * 4, frame.audio.count % 8 == 0, frame.sampleOffset >= lastAudioEnd else { throw CoCoRecordingError.message("The recording input lost its frame/audio ordering.") } let gap = frame.sampleOffset - lastAudioEnd guard gap <= Int64(Self.rate) * 2 else { throw CoCoRecordingError.message("The recording input was interrupted. The working movie has been retained.") } if gap > 0 { ingressLock.lock() let allowed = retainedSamples + Int(gap) <= Int(Self.rate) * 2 if allowed { retainedSamples += Int(gap) } ingressLock.unlock() guard allowed else { throw CoCoRecordingError.message("The recorder reached its bounded audio limit.") } sounds.append((Data(count: Int(gap) * 8), lastAudioEnd)); queuedSamples += Int(gap) } if !frame.audio.isEmpty { sounds.append((frame.audio, frame.sampleOffset)); queuedSamples += frame.audio.count / 8 } lastAudioEnd = frame.sampleOffset + Int64(frame.audio.count / 8) finalSample = max(finalSample, lastAudioEnd) let tick = frame.sampleOffset * 60 / Int64(Self.rate) if tick > lastVideoTick { let duplicates = Int(tick - lastVideoTick - 1) if duplicates > 0 { ingressLock.lock() let allowed = retainedFrames + duplicates <= 12 if allowed { retainedFrames += duplicates } ingressLock.unlock() guard allowed else { throw CoCoRecordingError.message("The recording frame clock was interrupted. The working movie is retained.") } let held = previousPicture ?? (frame.pixels, frame.area) for t in (lastVideoTick + 1)..) -> Void) { ingressLock.lock(); accepting = false; ingressLock.unlock() queue.async { [self] in if let finishResult { completion(finishResult); return } finishReplies.append(completion) guard !finishing, !failed else { return } finishing = true; drain() } } /// A disposable producer advances faster than wall time. Its own worker may /// await this acknowledgement rather than overflowing the unchanged bounded /// ingress queue. The application recorder continues using nonblocking append. func whenDrained(_ completion: @escaping @Sendable (Result) -> Void) { let deadline = ProcessInfo.processInfo.systemUptime + 10 func check() { queue.async { [self] in if failed { completion(.failure(writer.error ?? CoCoRecordingError.message("The video writer stopped."))); return } ingressLock.lock() let empty = retainedFrames == 0 && retainedSamples == 0 ingressLock.unlock() if empty { completion(.success(())) } else if ProcessInfo.processInfo.systemUptime >= deadline { let error = CoCoRecordingError.message("The video writer did not drain within its deadline.") fail(error); completion(.failure(error)) } else { queue.asyncAfter(deadline: .now() + .milliseconds(5), execute: check) } } } check() } private func drain() { guard !failed else { return } do { while !images.isEmpty && video.isReadyForMoreMediaData { let image = images.removeFirst() let pixel = try Self.pixelBuffer(image.0, area: image.1, pool: adaptor.pixelBufferPool) guard adaptor.append(pixel, withPresentationTime: CMTime(value: image.2, timescale: 60)) else { throw writer.error ?? CoCoRecordingError.message("A video frame could not be written.") } release(frames: 1, samples: 0) } while !sounds.isEmpty && audio.isReadyForMoreMediaData { let sound = sounds.removeFirst(); queuedSamples -= sound.0.count / 8 let sample = try audioSample(sound.0, offset: sound.1) guard audio.append(sample) else { throw writer.error ?? CoCoRecordingError.message("CoCo sound could not be written.") } release(frames: 0, samples: sound.0.count / 8) } if writer.status == .failed { throw writer.error ?? CoCoRecordingError.message("The movie writer stopped.") } if images.isEmpty && sounds.isEmpty { if finishing, !closingWriter { closingWriter = true writer.endSession(atSourceTime: CMTime(value: max(finalSample, 735), timescale: Self.rate)) video.markAsFinished(); audio.markAsFinished() writer.finishWriting { [self] in queue.async { [self] in if let failureError { finishAll(.failure(failureError)) } else if writer.status == .completed { finishAll(.success(workingURL)) } else { finishAll(.failure(writer.error ?? CoCoRecordingError.message("The working movie could not be finalized."))) } } } } } else if !draining { draining = true queue.asyncAfter(deadline: .now() + .milliseconds(5)) { [self] in draining = false; drain() } } } catch { fail(error) } } private func fail(_ error: Error) { guard !failed else { return }; failed = true failureError = error ingressLock.lock(); accepting = false; retainedFrames = 0; retainedSamples = 0; ingressLock.unlock() images.removeAll(); sounds.removeAll(); queuedSamples = 0 // Finishing what the framework can retain is preferable to cancelling // and deleting fragments. Every Stop waits for that closure, including // Stop arriving after the failure callback or during another Stop. if closingWriter { // The existing AVFoundation callback owns the terminal result. } else if writer.status == .writing { closingWriter = true video.markAsFinished(); audio.markAsFinished() writer.finishWriting { [self] in queue.async { [self] in finishAll(.failure(error)) } } } else { finishAll(.failure(error)) } failure?(error.localizedDescription) } private func finishAll(_ result: Result) { guard finishResult == nil else { return } finishResult = result let replies = finishReplies; finishReplies.removeAll() for reply in replies { reply(result) } } private func release(frames: Int, samples: Int) { ingressLock.lock(); retainedFrames -= frames; retainedSamples -= samples; ingressLock.unlock() } private func audioSample(_ data: Data, offset: Int64) throws -> CMSampleBuffer { var block: CMBlockBuffer? guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil, blockLength: data.count, blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0, dataLength: data.count, flags: 0, blockBufferOut: &block) == kCMBlockBufferNoErr, let block else { throw CoCoRecordingError.message("Audio buffer allocation failed.") } let copied = data.withUnsafeBytes { CMBlockBufferReplaceDataBytes(with: $0.baseAddress!, blockBuffer: block, offsetIntoDestination: 0, dataLength: data.count) } var time = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: Self.rate), presentationTimeStamp: CMTime(value: offset, timescale: Self.rate), decodeTimeStamp: .invalid) var sample: CMSampleBuffer? guard copied == noErr, CMSampleBufferCreateReady(allocator: kCFAllocatorDefault, dataBuffer: block, formatDescription: audioFormat, sampleCount: data.count / 8, sampleTimingEntryCount: 1, sampleTimingArray: &time, sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, let sample else { throw CoCoRecordingError.message("The CoCo audio sample could not be prepared.") } return sample } static func pixelBuffer(_ bytes: Data, area: CGRect, pool: CVPixelBufferPool?) throws -> CVPixelBuffer { guard bytes.count == 640 * 480 * 4 else { throw CoCoRecordingError.message("No complete CoCo picture is available.") } guard [area.minX, area.minY, area.width, area.height].allSatisfy(\.isFinite) else { throw CoCoRecordingError.message("The recording picture dimensions are invalid.") } let bounds = CGRect(x: 0, y: 0, width: 640, height: 480) let rect = area.intersection(bounds).integral guard !rect.isEmpty, !rect.isNull, let pool else { throw CoCoRecordingError.message("The recording picture dimensions are invalid.") } var buffer: CVPixelBuffer? guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess, let buffer else { throw CoCoRecordingError.message("A video pixel buffer could not be allocated.") } CVPixelBufferLockBaseAddress(buffer, []); defer { CVPixelBufferUnlockBaseAddress(buffer, []) } let color = CGColorSpace(name: CGColorSpace.sRGB)! guard let source = CGDataProvider(data: bytes as CFData), let image = CGImage(width: 640, height: 480, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: 640 * 4, space: color, bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue), provider: source, decode: nil, shouldInterpolate: false, intent: .defaultIntent)?.cropping(to: rect), let context = CGContext(data: CVPixelBufferGetBaseAddress(buffer), width: width, height: height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: color, bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else { throw CoCoRecordingError.message("The CoCo picture could not be rendered for video.") } context.setFillColor(CGColor(gray: 0, alpha: 1)); context.fill(CGRect(x: 0, y: 0, width: width, height: height)) let scale = min(CGFloat(width) / rect.width, CGFloat(height) / rect.height) let size = CGSize(width: rect.width * scale, height: rect.height * scale) context.interpolationQuality = .none context.draw(image, in: CGRect(x: (CGFloat(width)-size.width)/2, y: (CGFloat(height)-size.height)/2, width: size.width, height: size.height)) return buffer } static func exportMP4(from source: URL, to destination: URL) async throws { guard !FileManager.default.fileExists(atPath: destination.path), let export = AVAssetExportSession(asset: AVURLAsset(url: source), presetName: AVAssetExportPresetPassthrough), export.supportedFileTypes.contains(.mp4) else { throw CoCoRecordingError.message("The retained movie cannot be converted to this MP4 destination.") } export.outputURL = destination; export.outputFileType = .mp4 await export.export() guard export.status == .completed else { throw export.error ?? CoCoRecordingError.message("MP4 completion failed; the working movie has been retained.") } let asset = AVURLAsset(url: destination) guard try await !asset.loadTracks(withMediaType: .video).isEmpty, try await !asset.loadTracks(withMediaType: .audio).isEmpty, try await asset.load(.duration).seconds > 0 else { throw CoCoRecordingError.message("The completed movie did not contain playable video and audio.") } } }