Sync source edition with VCC Plus 3.7.0 build 63
This commit is contained in:
1 parent
c652a5276d
commit
b2ecec4d87
86 files changed
+10519
-472
No files matched your search
@@ -0,0 +1,300 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import CoreGraphics
|
||||
import CoreVideo
|
||||
import AudioToolbox
|
||||
|
||||
/// Only emulator pixels and generated stereo PCM enter this boundary. There is
|
||||
/// deliberately no host-window, microphone, or screen-recording API here.
|
||||
struct CoCoVideoFrame: Sendable {
|
||||
var pixels: Data
|
||||
var area: CGRect
|
||||
var audio: Data // packed, interleaved Float32 stereo, 44,100 sample frames/sec
|
||||
var sampleOffset: Int64
|
||||
}
|
||||
|
||||
enum CoCoRecordingError: LocalizedError {
|
||||
case message(String)
|
||||
var errorDescription: String? { if case let .message(s) = self { return s }; return nil }
|
||||
}
|
||||
|
||||
/// Serial, bounded writer. Working MOV fragments remain available after a
|
||||
/// failure; only a verified MP4 completion permits their owner to clean up.
|
||||
final class CoCoVideoEncoder: @unchecked Sendable {
|
||||
static let width = 1280, height = 960
|
||||
static let rate: Int32 = 44_100
|
||||
let workingURL: URL
|
||||
private let queue = DispatchQueue(label: "com.brianholman.vccplus.video-encoder", qos: .userInitiated)
|
||||
private let writer: AVAssetWriter
|
||||
private let video: AVAssetWriterInput
|
||||
private let audio: AVAssetWriterInput
|
||||
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
|
||||
private var audioFormat: CMAudioFormatDescription?
|
||||
private var images: [(Data, CGRect, Int64)] = []
|
||||
private var sounds: [(Data, Int64)] = []
|
||||
private var queuedSamples = 0
|
||||
private var lastVideoTick: Int64 = -1
|
||||
private var previousPicture: (Data, CGRect)?
|
||||
private var lastAudioEnd: Int64 = 0
|
||||
private var finalSample: Int64 = 0
|
||||
private var finishing = false, draining = false, failed = false
|
||||
private var finishReplies: [(Result<URL, Error>) -> Void] = []
|
||||
private var finishResult: Result<URL, Error>?
|
||||
private var closingWriter = false
|
||||
private var failureError: Error?
|
||||
private let ingressLock = NSLock()
|
||||
private var accepting = true
|
||||
private var retainedFrames = 0, retainedSamples = 0
|
||||
var failure: (@Sendable (String) -> Void)?
|
||||
/// Read-only AVFoundation lifecycle evidence; a finish reply promises this
|
||||
/// terminal state, including when a failed movie is retained for recovery.
|
||||
var writerIsTerminal: Bool { [.completed, .failed, .cancelled].contains(writer.status) }
|
||||
|
||||
init(workingURL: URL) throws {
|
||||
self.workingURL = workingURL
|
||||
writer = try AVAssetWriter(outputURL: workingURL, fileType: .mov)
|
||||
writer.movieFragmentInterval = CMTime(seconds: 10, preferredTimescale: 600)
|
||||
writer.initialMovieFragmentInterval = CMTime(seconds: 1, preferredTimescale: 600)
|
||||
video = AVAssetWriterInput(mediaType: .video, outputSettings: [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: Self.width,
|
||||
AVVideoHeightKey: Self.height, AVVideoCompressionPropertiesKey: [
|
||||
AVVideoAverageBitRateKey: 4_000_000, AVVideoExpectedSourceFrameRateKey: 60,
|
||||
AVVideoMaxKeyFrameIntervalKey: 60, AVVideoAllowFrameReorderingKey: false]])
|
||||
audio = AVAssetWriterInput(mediaType: .audio, outputSettings: [
|
||||
AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: 48_000,
|
||||
AVNumberOfChannelsKey: 2, AVEncoderBitRateKey: 160_000])
|
||||
video.expectsMediaDataInRealTime = true; audio.expectsMediaDataInRealTime = true
|
||||
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: video,
|
||||
sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey as String: Self.width, kCVPixelBufferHeightKey as String: Self.height,
|
||||
kCVPixelBufferCGImageCompatibilityKey as String: true,
|
||||
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true])
|
||||
guard writer.canAdd(video), writer.canAdd(audio) else { throw CoCoRecordingError.message("The native video/audio encoder is unavailable.") }
|
||||
writer.add(video); writer.add(audio)
|
||||
var format = AudioStreamBasicDescription(mSampleRate: Double(Self.rate), mFormatID: kAudioFormatLinearPCM,
|
||||
mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked,
|
||||
mBytesPerPacket: 8, mFramesPerPacket: 1, mBytesPerFrame: 8, mChannelsPerFrame: 2,
|
||||
mBitsPerChannel: 32, mReserved: 0)
|
||||
guard CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &format,
|
||||
layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil,
|
||||
extensions: nil, formatDescriptionOut: &audioFormat) == noErr,
|
||||
writer.startWriting() else { throw writer.error ?? CoCoRecordingError.message("The recording could not start.") }
|
||||
writer.startSession(atSourceTime: .zero)
|
||||
}
|
||||
|
||||
func append(_ frames: [CoCoVideoFrame]) {
|
||||
let samples = frames.reduce(0) { $0 + $1.audio.count / 8 }
|
||||
ingressLock.lock()
|
||||
guard accepting else { ingressLock.unlock(); return }
|
||||
guard retainedFrames + frames.count <= 12, retainedSamples + samples <= Int(Self.rate) * 2 else {
|
||||
accepting = false; ingressLock.unlock()
|
||||
queue.async { [self] in fail(CoCoRecordingError.message("The recorder reached its bounded input limit. The working movie has been retained.")) }
|
||||
return
|
||||
}
|
||||
retainedFrames += frames.count; retainedSamples += samples
|
||||
ingressLock.unlock()
|
||||
queue.async { [self] in
|
||||
guard !failed, !finishing else { return }
|
||||
do {
|
||||
for frame in frames {
|
||||
guard frame.pixels.count == 640 * 480 * 4, frame.audio.count % 8 == 0,
|
||||
frame.sampleOffset >= lastAudioEnd else {
|
||||
throw CoCoRecordingError.message("The recording input lost its frame/audio ordering.")
|
||||
}
|
||||
let gap = frame.sampleOffset - lastAudioEnd
|
||||
guard gap <= Int64(Self.rate) * 2 else { throw CoCoRecordingError.message("The recording input was interrupted. The working movie has been retained.") }
|
||||
if gap > 0 {
|
||||
ingressLock.lock()
|
||||
let allowed = retainedSamples + Int(gap) <= Int(Self.rate) * 2
|
||||
if allowed { retainedSamples += Int(gap) }
|
||||
ingressLock.unlock()
|
||||
guard allowed else { throw CoCoRecordingError.message("The recorder reached its bounded audio limit.") }
|
||||
sounds.append((Data(count: Int(gap) * 8), lastAudioEnd)); queuedSamples += Int(gap)
|
||||
}
|
||||
if !frame.audio.isEmpty { sounds.append((frame.audio, frame.sampleOffset)); queuedSamples += frame.audio.count / 8 }
|
||||
lastAudioEnd = frame.sampleOffset + Int64(frame.audio.count / 8)
|
||||
finalSample = max(finalSample, lastAudioEnd)
|
||||
let tick = frame.sampleOffset * 60 / Int64(Self.rate)
|
||||
if tick > lastVideoTick {
|
||||
let duplicates = Int(tick - lastVideoTick - 1)
|
||||
if duplicates > 0 {
|
||||
ingressLock.lock()
|
||||
let allowed = retainedFrames + duplicates <= 12
|
||||
if allowed { retainedFrames += duplicates }
|
||||
ingressLock.unlock()
|
||||
guard allowed else { throw CoCoRecordingError.message("The recording frame clock was interrupted. The working movie is retained.") }
|
||||
let held = previousPicture ?? (frame.pixels, frame.area)
|
||||
for t in (lastVideoTick + 1)..<tick { images.append((held.0, held.1, t)) }
|
||||
}
|
||||
images.append((frame.pixels, frame.area, tick)); lastVideoTick = tick
|
||||
previousPicture = (frame.pixels, frame.area)
|
||||
}
|
||||
else { release(frames: 1, samples: 0) }
|
||||
guard images.count <= 12, queuedSamples <= Int(Self.rate) * 2 else {
|
||||
throw CoCoRecordingError.message("The video encoder could not keep up. The working movie has been retained.")
|
||||
}
|
||||
}
|
||||
drain()
|
||||
} catch { fail(error) }
|
||||
}
|
||||
}
|
||||
|
||||
func finish(_ completion: @escaping (Result<URL, Error>) -> Void) {
|
||||
ingressLock.lock(); accepting = false; ingressLock.unlock()
|
||||
queue.async { [self] in
|
||||
if let finishResult { completion(finishResult); return }
|
||||
finishReplies.append(completion)
|
||||
guard !finishing, !failed else { return }
|
||||
finishing = true; drain()
|
||||
}
|
||||
}
|
||||
|
||||
/// A disposable producer advances faster than wall time. Its own worker may
|
||||
/// await this acknowledgement rather than overflowing the unchanged bounded
|
||||
/// ingress queue. The application recorder continues using nonblocking append.
|
||||
func whenDrained(_ completion: @escaping @Sendable (Result<Void, Error>) -> Void) {
|
||||
let deadline = ProcessInfo.processInfo.systemUptime + 10
|
||||
func check() {
|
||||
queue.async { [self] in
|
||||
if failed { completion(.failure(writer.error ?? CoCoRecordingError.message("The video writer stopped."))); return }
|
||||
ingressLock.lock()
|
||||
let empty = retainedFrames == 0 && retainedSamples == 0
|
||||
ingressLock.unlock()
|
||||
if empty { completion(.success(())) }
|
||||
else if ProcessInfo.processInfo.systemUptime >= deadline {
|
||||
let error = CoCoRecordingError.message("The video writer did not drain within its deadline.")
|
||||
fail(error); completion(.failure(error))
|
||||
} else { queue.asyncAfter(deadline: .now() + .milliseconds(5), execute: check) }
|
||||
}
|
||||
}
|
||||
check()
|
||||
}
|
||||
|
||||
private func drain() {
|
||||
guard !failed else { return }
|
||||
do {
|
||||
while !images.isEmpty && video.isReadyForMoreMediaData {
|
||||
let image = images.removeFirst()
|
||||
let pixel = try Self.pixelBuffer(image.0, area: image.1, pool: adaptor.pixelBufferPool)
|
||||
guard adaptor.append(pixel, withPresentationTime: CMTime(value: image.2, timescale: 60)) else {
|
||||
throw writer.error ?? CoCoRecordingError.message("A video frame could not be written.")
|
||||
}
|
||||
release(frames: 1, samples: 0)
|
||||
}
|
||||
while !sounds.isEmpty && audio.isReadyForMoreMediaData {
|
||||
let sound = sounds.removeFirst(); queuedSamples -= sound.0.count / 8
|
||||
let sample = try audioSample(sound.0, offset: sound.1)
|
||||
guard audio.append(sample) else { throw writer.error ?? CoCoRecordingError.message("CoCo sound could not be written.") }
|
||||
release(frames: 0, samples: sound.0.count / 8)
|
||||
}
|
||||
if writer.status == .failed { throw writer.error ?? CoCoRecordingError.message("The movie writer stopped.") }
|
||||
if images.isEmpty && sounds.isEmpty {
|
||||
if finishing, !closingWriter {
|
||||
closingWriter = true
|
||||
writer.endSession(atSourceTime: CMTime(value: max(finalSample, 735), timescale: Self.rate))
|
||||
video.markAsFinished(); audio.markAsFinished()
|
||||
writer.finishWriting { [self] in
|
||||
queue.async { [self] in
|
||||
if let failureError { finishAll(.failure(failureError)) }
|
||||
else if writer.status == .completed { finishAll(.success(workingURL)) }
|
||||
else { finishAll(.failure(writer.error ?? CoCoRecordingError.message("The working movie could not be finalized."))) }
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if !draining {
|
||||
draining = true
|
||||
queue.asyncAfter(deadline: .now() + .milliseconds(5)) { [self] in draining = false; drain() }
|
||||
}
|
||||
} catch { fail(error) }
|
||||
}
|
||||
|
||||
private func fail(_ error: Error) {
|
||||
guard !failed else { return }; failed = true
|
||||
failureError = error
|
||||
ingressLock.lock(); accepting = false; retainedFrames = 0; retainedSamples = 0; ingressLock.unlock()
|
||||
images.removeAll(); sounds.removeAll(); queuedSamples = 0
|
||||
// Finishing what the framework can retain is preferable to cancelling
|
||||
// and deleting fragments. Every Stop waits for that closure, including
|
||||
// Stop arriving after the failure callback or during another Stop.
|
||||
if closingWriter {
|
||||
// The existing AVFoundation callback owns the terminal result.
|
||||
} else if writer.status == .writing {
|
||||
closingWriter = true
|
||||
video.markAsFinished(); audio.markAsFinished()
|
||||
writer.finishWriting { [self] in queue.async { [self] in finishAll(.failure(error)) } }
|
||||
} else { finishAll(.failure(error)) }
|
||||
failure?(error.localizedDescription)
|
||||
}
|
||||
private func finishAll(_ result: Result<URL, Error>) {
|
||||
guard finishResult == nil else { return }
|
||||
finishResult = result
|
||||
let replies = finishReplies; finishReplies.removeAll()
|
||||
for reply in replies { reply(result) }
|
||||
}
|
||||
private func release(frames: Int, samples: Int) {
|
||||
ingressLock.lock(); retainedFrames -= frames; retainedSamples -= samples; ingressLock.unlock()
|
||||
}
|
||||
|
||||
private func audioSample(_ data: Data, offset: Int64) throws -> CMSampleBuffer {
|
||||
var block: CMBlockBuffer?
|
||||
guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil,
|
||||
blockLength: data.count, blockAllocator: kCFAllocatorDefault, customBlockSource: nil,
|
||||
offsetToData: 0, dataLength: data.count, flags: 0, blockBufferOut: &block) == kCMBlockBufferNoErr,
|
||||
let block else { throw CoCoRecordingError.message("Audio buffer allocation failed.") }
|
||||
let copied = data.withUnsafeBytes { CMBlockBufferReplaceDataBytes(with: $0.baseAddress!, blockBuffer: block, offsetIntoDestination: 0, dataLength: data.count) }
|
||||
var time = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: Self.rate),
|
||||
presentationTimeStamp: CMTime(value: offset, timescale: Self.rate), decodeTimeStamp: .invalid)
|
||||
var sample: CMSampleBuffer?
|
||||
guard copied == noErr, CMSampleBufferCreateReady(allocator: kCFAllocatorDefault,
|
||||
dataBuffer: block, formatDescription: audioFormat, sampleCount: data.count / 8,
|
||||
sampleTimingEntryCount: 1, sampleTimingArray: &time, sampleSizeEntryCount: 0,
|
||||
sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, let sample else {
|
||||
throw CoCoRecordingError.message("The CoCo audio sample could not be prepared.")
|
||||
}
|
||||
return sample
|
||||
}
|
||||
|
||||
static func pixelBuffer(_ bytes: Data, area: CGRect, pool: CVPixelBufferPool?) throws -> CVPixelBuffer {
|
||||
guard bytes.count == 640 * 480 * 4 else { throw CoCoRecordingError.message("No complete CoCo picture is available.") }
|
||||
guard [area.minX, area.minY, area.width, area.height].allSatisfy(\.isFinite) else {
|
||||
throw CoCoRecordingError.message("The recording picture dimensions are invalid.")
|
||||
}
|
||||
let bounds = CGRect(x: 0, y: 0, width: 640, height: 480)
|
||||
let rect = area.intersection(bounds).integral
|
||||
guard !rect.isEmpty, !rect.isNull, let pool else { throw CoCoRecordingError.message("The recording picture dimensions are invalid.") }
|
||||
var buffer: CVPixelBuffer?
|
||||
guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess, let buffer else { throw CoCoRecordingError.message("A video pixel buffer could not be allocated.") }
|
||||
CVPixelBufferLockBaseAddress(buffer, []); defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
|
||||
let color = CGColorSpace(name: CGColorSpace.sRGB)!
|
||||
guard let source = CGDataProvider(data: bytes as CFData),
|
||||
let image = CGImage(width: 640, height: 480, bitsPerComponent: 8, bitsPerPixel: 32,
|
||||
bytesPerRow: 640 * 4, space: color,
|
||||
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue),
|
||||
provider: source, decode: nil, shouldInterpolate: false, intent: .defaultIntent)?.cropping(to: rect),
|
||||
let context = CGContext(data: CVPixelBufferGetBaseAddress(buffer), width: width, height: height,
|
||||
bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: color,
|
||||
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else {
|
||||
throw CoCoRecordingError.message("The CoCo picture could not be rendered for video.")
|
||||
}
|
||||
context.setFillColor(CGColor(gray: 0, alpha: 1)); context.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
||||
let scale = min(CGFloat(width) / rect.width, CGFloat(height) / rect.height)
|
||||
let size = CGSize(width: rect.width * scale, height: rect.height * scale)
|
||||
context.interpolationQuality = .none
|
||||
context.draw(image, in: CGRect(x: (CGFloat(width)-size.width)/2, y: (CGFloat(height)-size.height)/2, width: size.width, height: size.height))
|
||||
return buffer
|
||||
}
|
||||
|
||||
static func exportMP4(from source: URL, to destination: URL) async throws {
|
||||
guard !FileManager.default.fileExists(atPath: destination.path),
|
||||
let export = AVAssetExportSession(asset: AVURLAsset(url: source), presetName: AVAssetExportPresetPassthrough),
|
||||
export.supportedFileTypes.contains(.mp4) else { throw CoCoRecordingError.message("The retained movie cannot be converted to this MP4 destination.") }
|
||||
export.outputURL = destination; export.outputFileType = .mp4
|
||||
await export.export()
|
||||
guard export.status == .completed else { throw export.error ?? CoCoRecordingError.message("MP4 completion failed; the working movie has been retained.") }
|
||||
let asset = AVURLAsset(url: destination)
|
||||
guard try await !asset.loadTracks(withMediaType: .video).isEmpty,
|
||||
try await !asset.loadTracks(withMediaType: .audio).isEmpty,
|
||||
try await asset.load(.duration).seconds > 0 else { throw CoCoRecordingError.message("The completed movie did not contain playable video and audio.") }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user