diff --git a/apple/Common/AV.swift b/apple/Common/AV.swift index fe44ae4..6a19074 100644 --- a/apple/Common/AV.swift +++ b/apple/Common/AV.swift @@ -24,7 +24,7 @@ func assert_av_output_queue() { class AV { static let shared = AV() - static let defaultAudioFormat = kAudioFormatMPEG4AAC + static let defaultAudioFormatID = kAudioFormatMPEG4AAC static let defaultAudioInterval = 0.1 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -61,6 +61,14 @@ class AV { } } + var defaultAudioInputFormat: AudioStreamBasicDescription? { + guard let inputFormat = AVAudioEngine().inputNode?.inputFormat(forBus: AudioBus.input) else { return nil } + + return AudioStreamBasicDescription.CreateVBR(AV.defaultAudioFormatID, + inputFormat.sampleRate/*8000*/, + 1/*inputFormat.channelCount*/) + } + private func _defaultVideoInput(_ id: IOID, _ rotated: Bool, _ session: inout AVCaptureSession.Accessor?, @@ -95,9 +103,11 @@ class AV { private func _defaultAudioInput(_ id: IOID, _ x: inout [IOSessionProtocol]) { + guard let format = defaultAudioInputFormat else { return } + let input = AudioInput( - AV.defaultAudioFormat, + format, AV.defaultAudioInterval, NetworkAACSerializer( NetworkOutputAudio(id))) @@ -153,21 +163,23 @@ class AV { return VideoFormat(dimensions) } } - - func defaultIOSync(_ sid: String) -> IOSync { - let result = activeIOSync[sid] + + func defaultIOSync(_ gid: String) -> IOSync { + let result = activeIOSync[gid] if result != nil { return result! } - activeIOSync[sid] = IOSync() + activeIOSync[gid] = IOSync() - return activeIOSync[sid]! + return activeIOSync[gid]! } func startAudioOutput(_ id: IOID, _ session: IOSessionProtocol) throws { + try defaultIOSync(id.gid).start() try session.start() + activeAudioOutput[id.from] = session } @@ -181,8 +193,8 @@ class AV { activeAudioOutput.removeAll() } - func defaultNetworkInputVideo(_ id: IOID, - _ output: VideoOutputProtocol) -> IODataProtocol { + func defaultNetworkVideoOutput(_ id: IOID, + _ output: VideoOutputProtocol) -> IODataProtocol { let time = VideoTimeDeserializer(H264Part.Time.rawValue) @@ -214,17 +226,26 @@ class AV { return result } - func defaultNetworkOutputAudio(_ id: IOID, + func defaultNetworkAudioOutput(_ id: IOID, _ format: AudioFormat, _ session: inout IOSessionProtocol?) -> IODataProtocol { let time = AudioTimeDeserializer(AACPart.NetworkPacket.rawValue) let output = - AudioOutput(format, AV.defaultAudioFormat, AV.defaultAudioInterval) + AudioOutput( + factory(format), + avOutputQueue) + + let decoder = + AudioDecoder( + factory(format), + output.format, + output) let sync = - defaultIOSync(id.gid) + defaultIOSync( + id.gid) let syncBus = IOSyncBus( @@ -242,17 +263,17 @@ class AV { IOKind.Audio, time, NetworkAACDeserializer( - output)) + decoder)) - session = output + session = create([output, decoder]) return result } - func setupDefaultNetworkInputAudio(_ platformSession: IOSessionProtocol?) { + func setupDefaultNetworkAudioOutput(_ platformSession: IOSessionProtocol?) { let audioSessionStart = { (_ id: IOID, format: AudioFormat) throws -> IODataProtocol in var session: IOSessionProtocol? = nil - let result = AV.shared.defaultNetworkOutputAudio(id, format, &session) + let result = AV.shared.defaultNetworkAudioOutput(id, format, &session) if platformSession != nil && session != nil { let shared = session! @@ -298,4 +319,68 @@ class AV { } } } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Playback + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + var defaultAudioInputUncompressedFormat: AudioStreamBasicDescription? { + guard let inputFormat = AVAudioEngine().inputNode?.inputFormat(forBus: AudioBus.input) else { return nil } + var format = inputFormat.streamDescription.pointee + + format.mChannelsPerFrame = 1 + return format + } + + func startAudioUncompressedPlayback() throws { + guard let format = defaultAudioInputUncompressedFormat else { return } + + let input = + AudioInput( + format, + AV.defaultAudioInterval) + + let output = + AudioOutput( + input.format, + AV.shared.audioCaptureQueue) + + input.output = + NetworkAACSerializer( + NetworkAACDeserializer( + output)) + + try audioCaptureQueue.sync { + try AV.shared.startInput(create([output, input])) + } + } + + func startAudioCompressedPlayback() throws { + guard let format = defaultAudioInputFormat else { return } + + let input = + AudioInput( + format, + AV.defaultAudioInterval) + + let output = + AudioOutput( + input.format, + AV.shared.audioCaptureQueue) + + let decoder = + AudioDecoder( + input.format, + output.format, + output) + + input.output = + NetworkAACSerializer( + NetworkAACDeserializer( + decoder)) + + try audioCaptureQueue.sync { + try AV.shared.startInput(create([input, output, decoder])) + } + } } diff --git a/apple/Common/Application.swift b/apple/Common/Application.swift index 0a3dd5a..4bd8613 100644 --- a/apple/Common/Application.swift +++ b/apple/Common/Application.swift @@ -5,6 +5,9 @@ import Fabric import Crashlytics class Application : AppleApplicationDelegate { + + static let kCompressedPlayback = false + static let kUncompressedPlayback = false static let kServerIP = "kServerIP" static let kVideoWidth = "kVideoWidth" @@ -37,5 +40,18 @@ class Application : AppleApplicationDelegate { AV.shared.defaultVideoDimension = CMVideoDimensions(width: Int32(videoWidth!)!, height: Int32(videoHeight!)!) } + + // playback testing + + checkIO { + + if Application.kCompressedPlayback { + try AV.shared.startAudioCompressedPlayback() + } + + if Application.kUncompressedPlayback { + try AV.shared.startAudioUncompressedPlayback() + } + } } } diff --git a/apple/Common/IO/Audio.swift b/apple/Common/IO/Audio.swift index 741a967..6ae3746 100644 --- a/apple/Common/IO/Audio.swift +++ b/apple/Common/IO/Audio.swift @@ -2,6 +2,10 @@ import AVFoundation import AudioToolbox +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Simple types +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + protocol AudioOutputProtocol { func process(_ data: AudioData) @@ -19,7 +23,7 @@ struct AudioBus { struct AudioData { var bytes: UnsafePointer! var bytesNum: UInt32 = 0 - var packetDesc: UnsafePointer! + var packetDesc: UnsafePointer? var packetNum: UInt32 = 0 var timeStamp: AudioTimeStamp! @@ -29,7 +33,7 @@ struct AudioData { init(_ bytes: UnsafePointer, _ bytesNum: UInt32, - _ packetDesc: UnsafePointer, + _ packetDesc: UnsafePointer?, _ packetNum: UInt32, _ timeStamp: AudioTimeStamp) { self.bytes = bytes @@ -40,32 +44,45 @@ struct AudioData { } } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// AudioFormat +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + struct AudioFormat { typealias Factory = () -> AudioFormat + private static let kFormatID = "kFormatID" private static let kFlags = "kFlags" private static let kSampleRate = "kSampleRate" private static let kChannelCount = "kChannelCount" private static let kFramesPerPacket = "kFramesPerPacket" - private static let kPacketMaxSize = "kPacketMaxSize" private(set) var data: [String: Any] - init(_ x: AudioStreamBasicDescription, _ packetMaxSize: UInt32) { + init(_ x: AudioStreamBasicDescription) { data = [String: Any]() + self.formatID = x.mFormatID self.flags = x.mFormatFlags self.sampleRate = x.mSampleRate self.channelCount = x.mChannelsPerFrame self.framesPerPacket = x.mFramesPerPacket - self.packetMaxSize = packetMaxSize } init(_ data: [String: Any]) { self.data = data } + var formatID: UInt32 { + get { + return data.keys.contains(AudioFormat.kFormatID) ? data[AudioFormat.kFormatID] as! UInt32 : 0 + } + set { + data[AudioFormat.kFormatID] = newValue + } + } + var flags: UInt32 { get { return data.keys.contains(AudioFormat.kFlags) ? data[AudioFormat.kFlags] as! UInt32 : 0 @@ -101,17 +118,18 @@ struct AudioFormat { data[AudioFormat.kFramesPerPacket] = newValue } } +} - var packetMaxSize: UInt32 { - get { - return data.keys.contains(AudioFormat.kPacketMaxSize) ? data[AudioFormat.kPacketMaxSize] as! UInt32 : 0 - } - set { - data[AudioFormat.kPacketMaxSize] = newValue - } +func factory(_ value: AudioFormat) -> AudioFormat.Factory { + return { () in + return value } } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Protocols adapters +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + class AudioPipe : AudioOutputProtocol { var next: AudioOutputProtocol? @@ -138,7 +156,7 @@ class AudioTimeDeserializer : IOTimeProtocol { return UnsafePointer(x) } - func audioTime(_ packets: [Int: NSData], _ time: UnsafePointer) { + func audioTime(_ packets: inout [Int: NSData], _ time: UnsafePointer) { memcpy(UnsafeMutableRawPointer(mutating: packets[packetKey]!.bytes), time, MemoryLayout.size) @@ -151,7 +169,74 @@ class AudioTimeDeserializer : IOTimeProtocol { func time(_ data: inout [Int: NSData], _ time: Double) { let audioTime = UnsafeMutablePointer(mutating: self.audioTime(data)) audioTime.pointee.seconds(time) - self.audioTime(data, audioTime) + self.audioTime(&data, audioTime) } } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// AudioDataBuffer +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class AudioDataReader { + + var data = [AudioData]() + var capacity: Int + var current: AudioData? + var currentIndex: Int = 0 + + init(capacity: Int) { + self.capacity = capacity + } + + func push(_ data: AudioData) { + self.data.append(data) + + if self.data.count > capacity { + self.data.removeLast() + } + } + + private func popFirst() -> AudioData? { + let result = data.first + + if data.count != 0 { + data.removeFirst() + } + + return result + } + + func pop(_ count: Int, _ outData: UnsafeMutableRawPointer) { + + if current == nil { + current = popFirst() + currentIndex = 0 + } + + if current == nil { + memset(outData, 0, count) + return + } + + var countRead = min(count, Int(current!.bytesNum) - currentIndex) + var outIndex = 0 + + while current != nil && countRead > 0 { + memcpy(outData.advanced(by: outIndex), current?.bytes.advanced(by: currentIndex), countRead) + currentIndex += countRead + outIndex += countRead + + if currentIndex == Int(current!.bytesNum) { + current = popFirst() + currentIndex = 0 + } + + if current != nil { + countRead = min(count - outIndex, Int(current!.bytesNum) - currentIndex) + } + else { + memset(outData.advanced(by: outIndex), 0, count - outIndex) + } + } + } +} diff --git a/apple/Common/IO/AudioDecoder.swift b/apple/Common/IO/AudioDecoder.swift new file mode 100644 index 0000000..e05ce31 --- /dev/null +++ b/apple/Common/IO/AudioDecoder.swift @@ -0,0 +1,127 @@ +// +// AudioDecoderAAC.swift +// Chat +// +// Created by Ivan Khvorostinin on 15/06/2017. +// Copyright © 2017 ys1382. All rights reserved. +// + +import AudioToolbox + +class AudioDecoder : AudioOutputProtocol, IOSessionProtocol { + + var input: AudioFormat.Factory + var output: AudioStreamBasicDescription.Factory + let next: AudioOutput? + + var converter: AudioConverterRef? + var inputDescription: AudioStreamBasicDescription? + var outputDescription: AudioStreamBasicDescription? + var asc: UInt8 = 0 + + init(_ input: @escaping AudioFormat.Factory, + _ output: @escaping AudioStreamBasicDescription.Factory, + _ next: AudioOutput?) { + self.input = input + self.output = output + self.next = next + } + + func start() throws { + guard var outputDescription = try self.output() else { return } + outputDescription.mChannelsPerFrame = 1 + + self.inputDescription = AudioStreamBasicDescription.CreateVBR(self.input()) + self.outputDescription = outputDescription + + try checkStatus(AudioConverterNew(&inputDescription!, &outputDescription, &converter), + "decoder: AudioConverterNew") + } + + func stop() { + guard let converter = self.converter else { return } + + do { + try checkStatus(AudioConverterDispose(converter), + "AudioConverterDispose") + self.converter = nil + } + catch { + logIOError(error) + } + } + + func process(_ data: AudioData) { + + guard let converter = self.converter else { return } + + // asc + + if (asc == 0 && data.bytesNum == 2) { + logIO("asc size \(data.bytesNum)") + memcpy(&asc, data.bytes, 2); + return; + } + // adts + + else if (data.bytesNum == 7 || data.bytesNum == 9) { + logIO("adts size \(data.bytesNum)") + return; + } + + var pcmPacketNum = inputDescription!.mFramesPerPacket * data.packetNum + let pcmDataSize = pcmPacketNum * outputDescription!.mBytesPerPacket + + var pcmBufferList = AudioBufferList(mNumberBuffers: 1, + mBuffers: AudioBuffer(mNumberChannels: 1, + mDataByteSize: pcmDataSize, + mData: malloc(Int(pcmDataSize)))) + + do { + var dataCopy = data + + try checkStatus(AudioConverterFillComplexBuffer(converter, + decodeProc, + &dataCopy, + &pcmPacketNum, + &pcmBufferList, + nil), + "AudioConverterFillComplexBuffer") + } + catch { + logIOError(error) + } + + next?.process(AudioData(UnsafeMutablePointer(OpaquePointer((pcmBufferList.mBuffers.mData!))), + pcmDataSize, + nil, + pcmPacketNum, + data.timeStamp)) + } + + private let decodeProc: AudioConverterComplexInputDataProc = {( + converter: AudioConverterRef, + ioNumberDataPackets: UnsafeMutablePointer, + ioData: UnsafeMutablePointer, + outDataPacketDescription: UnsafeMutablePointer?>?, + userData: UnsafeMutableRawPointer?) -> OSStatus in + + let data = UnsafePointer(OpaquePointer(userData!)) + + ioNumberDataPackets.pointee = data.pointee.packetNum + + var bufferList = AudioBufferList() + + bufferList.mNumberBuffers = 1 + bufferList.mBuffers.mData = UnsafeMutableRawPointer(OpaquePointer(data.pointee.bytes)) + bufferList.mBuffers.mDataByteSize = data.pointee.bytesNum + bufferList.mBuffers.mNumberChannels = 1 + + var packetDescriptions = UnsafeMutablePointer(mutating: data.pointee.packetDesc) + + ioData.initialize(to: bufferList) + outDataPacketDescription?.initialize(to: packetDescriptions) + + return 0 + } +} diff --git a/apple/Common/IO/AudioInput.swift b/apple/Common/IO/AudioInput.swift index bd53fa6..268ec8a 100644 --- a/apple/Common/IO/AudioInput.swift +++ b/apple/Common/IO/AudioInput.swift @@ -4,14 +4,16 @@ import AVFoundation class AudioInput : NSObject, IOSessionProtocol { - private var audioFormat: AudioFormat? + private static let kBuffersCount = 3 + + public var output: AudioOutputProtocol? private var queue: AudioQueueRef? - private var buffer: AudioQueueBufferRef? + private var buffers = [AudioQueueBufferRef]() private var stopping: Bool = false - private let output: AudioOutputProtocol? - private let formatID: UInt32 + private var formatDescription: AudioStreamBasicDescription + private var formatChat: AudioFormat? private let interval: Double private var thread: ChatThread? @@ -19,83 +21,86 @@ class AudioInput : NSObject, IOSessionProtocol public var format: AudioFormat.Factory { get { - return { () in self.audioFormat! } + return { () in self.formatChat! } } } - init(_ formatID: UInt32, _ interval: Double, _ queue: DispatchQueue, _ output: AudioOutputProtocol?) { - self.output = output - self.formatID = formatID + init(_ format: AudioStreamBasicDescription, + _ interval: Double, + _ queue: DispatchQueue, + _ output: AudioOutputProtocol?) { + self.formatDescription = format self.interval = interval self.dqueue = queue + self.output = output + } + + convenience init(_ format: AudioStreamBasicDescription, + _ interval: Double, + _ output: AudioOutputProtocol?) { + self.init(format, interval, AV.shared.audioCaptureQueue, output) } - convenience init(_ formatID: UInt32, _ interval: Double, _ output: AudioOutputProtocol?) { - self.init(formatID, interval, AV.shared.audioCaptureQueue, output) + convenience init(_ format: AudioStreamBasicDescription, + _ interval: Double) { + self.init(format, interval, nil) } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // IOSessionProtocol //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - var tap: AudioQueueProcessingTapRef? func start() throws { assert(dqueue) // thread - thread = ChatThread("Audio Input") + thread = ChatThread(AudioInput.self) thread!.start() - // prepare format - - let engine = AVAudioEngine() - let inputFormat = engine.inputNode!.inputFormat(forBus: AudioBus.input) - var format = AudioStreamBasicDescription.CreateInput(formatID, - 8000/*inputFormat.sampleRate*/, - 1/*inputFormat.channelCount*/) - var packetMaxSize: UInt32 = 0 - // start queue + var packetMaxSize: UInt32 = 0 var bufferByteSize: UInt32 var size: UInt32 // create the queue try checkStatus(AudioQueueNewInput( - &format, + &formatDescription, callback, Unmanaged.passUnretained(self).toOpaque() /* userData */, thread!.runLoop.getCFRunLoop(), CFRunLoopMode.defaultMode.rawValue, 0 /* flags */, &queue), "AudioQueueNewInput failed") - // get the record format back from the queue's audio converter -- // the file may require a more specific stream description than was necessary to create the encoder. size = UInt32(MemoryLayout.size) try checkStatus(AudioQueueGetProperty(queue!, kAudioQueueProperty_StreamDescription, - &format, + &formatDescription, &size), "couldn't get queue's format"); // allocate and enqueue buffers - bufferByteSize = computeBufferSize(format, + bufferByteSize = computeBufferSize(formatDescription, interval, &packetMaxSize); // enough bytes for kBufferDurationSeconds - - try checkStatus(AudioQueueAllocateBuffer(queue!, - bufferByteSize, - &buffer), "AudioQueueAllocateBuffer failed"); - - try checkStatus(AudioQueueEnqueueBuffer(queue!, - buffer!, - 0, - nil), "AudioQueueEnqueueBuffer failed"); + + for _ in 0 ..< AudioInput.kBuffersCount { + var buffer: AudioQueueBufferRef? + + try checkStatus(AudioQueueAllocateBuffer(queue!, + bufferByteSize, + &buffer), "AudioQueueAllocateBuffer failed"); + + try checkStatus(AudioQueueEnqueueBuffer(queue!, + buffer!, + 0, + nil), "AudioQueueEnqueueBuffer failed"); + } // start the queue @@ -104,7 +109,7 @@ class AudioInput : NSObject, IOSessionProtocol // audio format - audioFormat = AudioFormat(format, packetMaxSize) + formatChat = AudioFormat(formatDescription) } func stop() { @@ -116,7 +121,6 @@ class AudioInput : NSObject, IOSessionProtocol thread?.sync { do { - // end recording try checkStatus(AudioQueueStop(queue, true), "AudioQueueStop failed") @@ -141,55 +145,6 @@ class AudioInput : NSObject, IOSessionProtocol // Utils //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - private func computeBufferSize(_ format: AudioStreamBasicDescription, - _ interval: Double, - _ packetMaxSize: inout UInt32) -> UInt32 - { - var packets: UInt32 - var frames: UInt32 - var bytes: UInt32 = 0 - - do { - frames = UInt32(ceil(interval * format.mSampleRate)) - - if (format.mBytesPerFrame > 0) { - bytes = frames * format.mBytesPerFrame - packetMaxSize = format.mBytesPerPacket - } - else { - if (format.mBytesPerPacket > 0) { - packetMaxSize = format.mBytesPerPacket // constant packet size - } - else { - var propertySize: UInt32 = UInt32(MemoryLayout.size) - try checkStatus(AudioQueueGetProperty(queue!, - kAudioQueueProperty_MaximumOutputPacketSize, - &packetMaxSize, - &propertySize), - "couldn't get queue's maximum output packet size") - } - - if (format.mFramesPerPacket > 0) { - packets = frames / format.mFramesPerPacket - } - else { - packets = frames // worst-case scenario: 1 frame in a packet - } - - if (packets == 0) { // sanity check - packets = 1 - } - - bytes = packets * packetMaxSize; - } - } - catch { - logIOError(error) - } - - return bytes; - } - private let callback: AudioQueueInputCallback = { (inUserData: UnsafeMutableRawPointer?, inAQ: AudioQueueRef, @@ -202,8 +157,7 @@ class AudioInput : NSObject, IOSessionProtocol guard input.stopping == false else { return } - - logIO("audio \(inStartTime.pointee.seconds)") + logIO("audio input \(inStartTime.pointee.seconds())") do { guard let queue = input.queue else { return } @@ -226,6 +180,8 @@ class AudioInput : NSObject, IOSessionProtocol MemoryLayout.size * Int(inNumPackets)) } + // process input + AV.shared.audioCaptureQueue.async { input.output!.process(AudioData(bytes, bytesSize, @@ -233,10 +189,25 @@ class AudioInput : NSObject, IOSessionProtocol inNumPackets, time)) } + + // simulate gaps + +// DispatchQueue.global().async { +// let x = arc4random_uniform(1000000) +// print("sleep \(Double(x) / 1000000.0)") +// usleep(x) +// AV.shared.audioCaptureQueue.async { +// input.output!.process(AudioData(bytes, +// bytesSize, +// packetDesc, +// inNumPackets, +// time)) +// } +// } } try checkStatus(AudioQueueEnqueueBuffer(input.queue!, - input.buffer!, + inBuffer, 0, nil), "AudioQueueEnqueueBuffer failed"); } @@ -244,4 +215,58 @@ class AudioInput : NSObject, IOSessionProtocol logIOError(error) } } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Utils + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + private func computeBufferSize(_ format: AudioStreamBasicDescription, + _ interval: Double, + _ packetMaxSize: inout UInt32) -> UInt32 + { + var packets: UInt32 + var frames: UInt32 + var bytes: UInt32 = 0 + + do { + frames = UInt32(ceil(interval * format.mSampleRate)) + + if (format.mBytesPerFrame > 0) { + bytes = frames * format.mBytesPerFrame + packetMaxSize = format.mBytesPerPacket + } + else { + if (format.mBytesPerPacket > 0) { + packetMaxSize = format.mBytesPerPacket // constant packet size + } + else { + var propertySize: UInt32 = UInt32(MemoryLayout.size) + try checkStatus(AudioQueueGetProperty(queue!, + kAudioQueueProperty_MaximumOutputPacketSize, + &packetMaxSize, + &propertySize), + "couldn't get queue's maximum output packet size") + } + + if (format.mFramesPerPacket > 0) { + packets = frames / format.mFramesPerPacket + } + else { + packets = frames // worst-case scenario: 1 frame in a packet + } + + if (packets == 0) { // sanity check + packets = 1 + } + + bytes = packets * packetMaxSize; + } + } + catch { + logIOError(error) + } + + return bytes; + } + } diff --git a/apple/Common/IO/AudioOutput.swift b/apple/Common/IO/AudioOutput.swift index 93c9bff..6dd76ec 100644 --- a/apple/Common/IO/AudioOutput.swift +++ b/apple/Common/IO/AudioOutput.swift @@ -7,24 +7,29 @@ import AudioToolbox class AudioOutput : AudioOutputProtocol, IOSessionProtocol { - private var queue: AudioQueueRef? - private var buffer: AudioQueueBufferRef? - private var packetsToRead: UInt32 = 0 private let dqueue: DispatchQueue + private var squeue: DispatchQueue? + private var unit: AppleAudioUnit? - let format: AudioFormat - let formatID: UInt32 - let interval: Double - - init(_ format: AudioFormat, _ formatID: UInt32, _ interval: Double, _ queue: DispatchQueue) { - self.format = format - self.formatID = formatID - self.interval = interval + private var buffer = AudioDataReader(capacity: 2) + private var bufferFrames = 0 + + private let formatInput: AudioFormat.Factory + private var formatDescription: AudioStreamBasicDescription? + + var packets: Int = 0 + + init(_ format: @escaping AudioFormat.Factory, _ queue: DispatchQueue) { self.dqueue = queue + self.formatInput = format } - convenience init(_ format: AudioFormat, _ formatID: UInt32, _ interval: Double) { - self.init(format, formatID, interval, AV.shared.avOutputQueue) + var format: AudioStreamBasicDescription.Factory { + get { + return { () in + return self.formatDescription + } + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -35,49 +40,24 @@ class AudioOutput : AudioOutputProtocol, IOSessionProtocol { assert(dqueue) - var format = AudioStreamBasicDescription.CreateOutput(self.format, formatID) - do { - // create queue - - try checkStatus(AudioQueueNewOutput(&format, - callback, - Unmanaged.passUnretained(self).toOpaque(), - CFRunLoopGetMain(), - CFRunLoopMode.commonModes.rawValue, - 0, - &queue), "AudioQueueNew failed") - - // we need to calculate how many packets we read at a time, and how big a buffer we need - // we base this on the size of the packets in the file and an approximate duration for each buffer - // first check to see what the max size of a packet is - if it is bigger - // than our allocation default size, that needs to become larger - - // adjust buffer size to represent about a half second of audio based on this format - var bufferByteSize: UInt32 = 0 - - _calculateBytesForTime (format, - self.format.packetMaxSize, - interval, - &bufferByteSize, - &packetsToRead) + packets = 0 + squeue = DispatchQueue.CreateCheckable("chat.AudioOutput") - let isFormatVBR = format.mBytesPerPacket == 0 || format.mFramesPerPacket == 0 + var callback = AURenderCallbackStruct(inputProc: self.callback, + inputProcRefCon: Unmanaged.passUnretained(self).toOpaque()) - try checkStatus(AudioQueueAllocateBufferWithPacketDescriptions(queue!, - bufferByteSize, - isFormatVBR ? packetsToRead : 0, - &buffer), "AudioQueueAllocateBuffer failed") - - // set the volume of the queue - try checkStatus(AudioQueueSetParameter(queue!, - kAudioQueueParam_Volume, - 1.0), "Set queue volume failed"); - - // start queue - - try checkStatus(AudioQueueStart(queue!, - nil), "AudioQueueStart failed") + #if os(iOS) + unit = try AppleAudioUnit(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO) + #else + unit = try AppleAudioUnit(kAudioUnitType_Output, kAudioUnitSubType_VoiceProcessingIO) + #endif + + try unit!.getFormat(kAudioUnitScope_Input, AudioBus.input, &formatDescription) + try unit!.setIOEnabled(kAudioUnitScope_Input, AudioBus.output, true) + try unit!.setRenderer(kAudioUnitScope_Input, AudioBus.input, &callback) + try unit!.initialize() + try unit!.start() } catch { logIOError(error) @@ -87,18 +67,10 @@ class AudioOutput : AudioOutputProtocol, IOSessionProtocol { func stop() { assert(dqueue) - - guard let queue = self.queue else { assert(false); return } do { - try checkStatus(AudioQueueStop(queue, - true), "AudioQueueStop failed") - - try checkStatus(AudioQueueDispose(queue, - true), "AudioQueueDispose failed") - - self.queue = nil - self.buffer = nil + try unit!.stop() + squeue = nil } catch { logIOError(error) @@ -108,68 +80,42 @@ class AudioOutput : AudioOutputProtocol, IOSessionProtocol { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // AudioOutputProtocol //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - + func process(_ data: AudioData) { - assert(dqueue) - do { - memcpy(self.buffer!.pointee.mAudioData, data.bytes, Int(data.bytesNum)) - self.buffer!.pointee.mAudioDataByteSize = data.bytesNum - - try checkStatus(AudioQueueEnqueueBuffer(self.queue!, - self.buffer!, - data.packetNum, - data.packetDesc), "AudioQueueEnqueueBuffer failed") - } - catch { - logIOError(error) + // first few packets plays with artefacts, so skip it + packets += 1 + guard packets > 10 else { return } + + squeue!.sync { + buffer.push(data) } } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Utils - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + var lastTime: Double = 0 + + private let callback: AURenderCallback = {( + inRefCon: UnsafeMutableRawPointer, + ioActionFlags: UnsafeMutablePointer, + inTimeStamp: UnsafePointer, + inBusNumber: UInt32, + inNumberFrames: UInt32, + ioData: UnsafeMutablePointer?) in - private func _calculateBytesForTime(_ inDesc: AudioStreamBasicDescription, - _ inpacketMaxSize: UInt32, - _ inSeconds: Double, - _ outBufferSize: inout UInt32, - _ outNumPackets: inout UInt32) - { - // we only use time here as a guideline - // we're really trying to get somewhere between 16K and 64K buffers, - // but not allocate too much if we don't need it - let maxBufferSize: UInt32 = 0x10000; // limit size to 64K - let minBufferSize: UInt32 = 0x4000; // limit size to 16K + let SELF = Unmanaged.fromOpaque(inRefCon).takeUnretainedValue() + let buffers = UnsafeBufferPointer(start: &ioData!.pointee.mBuffers, + count: Int(ioData!.pointee.mNumberBuffers)) - if (inDesc.mFramesPerPacket != 0) { - let numPacketsForTime = inDesc.mSampleRate / Double(inDesc.mFramesPerPacket) * inSeconds - outBufferSize = UInt32(numPacketsForTime * Double(inpacketMaxSize)) - } - else { - // if frames per packet is zero, then the codec has no predictable packet == time - // so we can't tailor this (we don't know how many Packets represent a time period - // we'll just return a default buffer size - outBufferSize = maxBufferSize > inpacketMaxSize ? maxBufferSize : inpacketMaxSize; - } - - // we're going to limit our size to our default - if (outBufferSize > maxBufferSize && outBufferSize > inpacketMaxSize) { - outBufferSize = maxBufferSize - } - else if (outBufferSize < minBufferSize) { - // also make sure we're not too small - we don't want to go the disk for too small chunks - outBufferSize = minBufferSize; + SELF.squeue!.sync { + SELF.buffer.pop(Int(inNumberFrames * SELF.formatDescription!.mBytesPerFrame), + buffers[0].mData!) + + for i in 1 ..< Int(ioData!.pointee.mNumberBuffers) { + memcpy(buffers[i].mData!, buffers[0].mData!, Int(buffers[0].mDataByteSize)) + } } - outNumPackets = outBufferSize / inpacketMaxSize; - } - - private let callback: AudioQueueOutputCallback = { - (inUserData: UnsafeMutableRawPointer?, - inAQ: AudioQueueRef, - inBuffer: AudioQueueBufferRef) in - + return 0 } } diff --git a/apple/Common/IO/IO.swift b/apple/Common/IO/IO.swift index baf10fe..e24406b 100644 --- a/apple/Common/IO/IO.swift +++ b/apple/Common/IO/IO.swift @@ -128,6 +128,11 @@ class IOTimebase : IODataProtocol { zero = dataTime } + else if zero! > dataTime { + assert(false) + return + } + time.time(©, timebase + dataTime - zero!) next?.process(copy) } @@ -159,6 +164,15 @@ func checkStatus(_ status: OSStatus, _ message: String) throws { } } +func checkIO(_ x: FuncVVT) { + do { + try x() + } + catch { + logIOError(error) + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // IOData dispatcher //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/apple/Common/IO/IOSync.swift b/apple/Common/IO/IOSync.swift new file mode 100644 index 0000000..5855bb9 --- /dev/null +++ b/apple/Common/IO/IOSync.swift @@ -0,0 +1,410 @@ + +import Foundation +import CoreMedia + +func logIOSync(_ message: String) { +// print("Sync: \(message)") +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// IOSyncBus +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class IOSyncBus : IODataProtocol { + + private let kind: IOKind + private let sync: IOSync + + init(_ kind: IOKind, _ sync: IOSync) { + self.kind = kind + self.sync = sync + } + + func process(_ data: [Int : NSData]) { + sync.process(kind, data) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// IOSync +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class IOSync : IOSessionProtocol { + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Internal Structs + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + private struct _TimerItem { + + let id: Int + let kind: IOKind + let data: [Int : NSData] + + init(_ id: Int, _ kind: IOKind, _ data: [Int : NSData]) { + self.id = id + self.kind = kind + self.data = data + } + } + + private struct _QueueItem { + var timer: Timer + let data: _TimerItem + + init(_ timer: Timer, _ data: _TimerItem) { + self.timer = timer + self.data = data + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Fields + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + private var started = false + + private var id: Int = 0 + private var nextID: Int { get { id += 1; return id } } + + private var output = [IOKind: IODataProtocol?]() + private var timing = [IOKind: IOTimeProtocol]() + + private let gap: IOSyncGap + + private var thread: ChatThread? + private var queue = [Int: _QueueItem]() + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Interface + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + init(_ gap: IOSyncGap) { + self.gap = gap + } + + init() { + self.gap = IOSyncGap() + } + + func add(_ kind: IOKind, _ time: IOTimeProtocol, _ output: IODataProtocol?) { + self.output[kind] = output + self.timing[kind] = time + } + + func start() throws { + guard started == false else { return } + + thread = ChatThread(IOSync.self) + thread!.start() + started = true + } + + func stop() { + thread!.cancel() + thread = nil + started = false + } + + func process(_ kind: IOKind, _ data: [Int : NSData]) { + output[kind]!?.process(data) +// _enqueue(kind, data) + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Sheduling + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + private func _enqueue(_ kind: IOKind, _ data: [Int : NSData]) { + thread!.sync({ + let id = self.nextID + let remoteTime = self.timing[kind]!.time(data) + let timerItem = _TimerItem(id, kind, data) + + let shedule: IOSyncGap.Shedule = { (_ time: Double) in + self._shedule(timerItem, time) + logIOSync("Sheduling \(kind) data with id \(id)") + } + + let reshedule: IOSyncGap.Shedule = { (_ shift: Double) in + self._reshedule(shift) + logIOSync("Resheduling with shift \(shift)") + } + + let zombie: FuncVV = { + logIOSync("Belated \(kind) data with id \(id) lost") + } + + self.gap.process(remoteTime, shedule: shedule, reshedule: reshedule, zombie: zombie) + }) + } + + private func _timer(_ at: Double, _ id: Int) -> Timer { + return Timer(fireAt: Date(timeIntervalSince1970: at), + interval: 0, + target: self, + selector: #selector(_output(timer:)), + userInfo: id, + repeats: false) + } + + private func _shedule(_ data: _TimerItem, _ at: Double) { + let timer = _timer(at, data.id) + queue[id] = _QueueItem(timer, data) + sheduleTimer(timer) + } + + private func _reshedule(_ shift: Double) { + for var i in queue { + i.value.timer.invalidate() + i.value.timer = _timer(i.value.timer.fireDate.addingTimeInterval(shift).timeIntervalSince1970, + i.value.data.id) + sheduleTimer(i.value.timer) + } + } + + @objc private func _output(timer: Timer) { + let id = timer.userInfo as! Int + + _output(queue[id]!.data) + queue.removeValue(forKey: id) + } + + private func _output(_ x: _TimerItem) { + + AV.shared.avOutputQueue.async { + self.output[x.kind]!?.process(x.data) + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Test support + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + fileprivate func sheduleTimer(_ timer: Timer) { + thread!.runLoop.add(timer, forMode: .defaultRunLoopMode) + } +} + +class IOSyncGap { + + // Shedule argument - local time to shedule + // Reshedule argument - time shift in seconds for sheduled data + typealias Shedule = (Double) -> Void + + private var localZero: Double? + private var remoteZero: Double? + private var remoteLast = 0.0 + private var shedule = [Int64: Double]() // local nanoseconds : remote seconds + + private var gap = 0.0 + private var gapMax: Int = 30 + private var gapLog = [Double]() + + var packets: Int = 0 + + var localTime: Double { + get { + return Date().timeIntervalSince1970 + } + } + + func process(_ remoteTime: Double, + shedule: @escaping Shedule, + reshedule: @escaping Shedule, + zombie: @escaping FuncVV) { + let localTime = self.localTime + var callback: FuncVV? + + packets += 1 + + // init + + if localZero == nil { + localZero = localTime + remoteZero = remoteTime + } + + // calc gap + + let gap = (localTime - localZero!) - (remoteTime - remoteZero!) + let gapPrev = self.gap + let sheduleTime = localTime + self.gap - gap + + _updateGap(gap) + + // sheduling + + let shedule_: Shedule = { (time: Double) in + shedule(time) + self.shedule[seconds2nano(sheduleTime)] = remoteTime + } + + // belated + + if _belated(remoteTime, localTime) { + callback = { zombie() } + } + + // reshedule + shedule + + else if micro(gap) > micro(self.gap) { + callback = { reshedule(self.gap - gapPrev); shedule_(sheduleTime) } + } + + // shedule + + else { + callback = { shedule_(sheduleTime) } + } + + if packets > 10 { + callback!() + } + } + + private func _belated(_ remoteTime: Double, _ localTime: Double) -> Bool { + + let localNano = seconds2nano(localTime) + + for i in shedule.keys { + if i > localNano { + continue + } + + if remoteLast < shedule[i]! { + remoteLast = shedule[i]! + } + + shedule.removeValue(forKey: i) + } + + return remoteTime < remoteLast + } + + private func _calc() -> Double { + let gapSorted = gapLog.sorted() + let diffAverage = (gapSorted.last! - gapSorted.first!) / Double(gapSorted.count) + var index: Int = gapSorted.count - 1 + + while index >= gapSorted.count * 10 / 100 && index > 1 { + if gapSorted[index] - gapSorted[index - 1] < diffAverage { + break + } + + index -= 1 + } + + return gapSorted[index] + } + + private func _updateGap(_ gap: Double) { + gapLog.append(gap) + + if gapLog.count > gapMax { + gapLog.removeFirst() + } + + self.gap = _calc() + + print("gap \(self.gap)") + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// IOSyncTest +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class IOSyncTest : IOSync { + + let gap = IOSyncTestGap() + let time = IOSyncTestTime() + + let zeroLocal = 0.0 + let zeroRemote = 1000.0 + var timeLocal: Double = 0 + var timeShedule: Double = 0 + + var success = true + + override init() { + super.init(gap) + + add(.Audio, time, nil) + add(.Video, time, nil) + } + + func test() -> Bool { + try! start() + return success + } + + override func start() throws { + try super.start() + + _test(.Video, 0.05, 0.2) // 0.07 + _test(.Video, 0.10, 0.2) // 0.12 + _test(.Audio, 0.10, 0.3) // 0.13 + _test(.Video, 0.15, 0.2) // 0.17 + _test(.Audio, 0.20, 0.2) // 0.22 + _test(.Video, 0.21, 0.2) // 0.23 + _test(.Audio, 0.30, 0.2) // 0.32 + _test(.Video, 0.30, 0.2) // 0.32 + _test(.Video, 0.25, 0.9) // 0.34 + _test(.Video, 0.35, 0.2) // 0.37 + _test(.Video, 0.39, 0.2) // 0.41 + _test(.Audio, 0.40, 0.2) // 0.42 + _test(.Video, 0.45, 0.2) // 0.47 + _test(.Audio, 0.50, 0.2) // 0.52 + } + + override func stop() { + super.stop() + } + + override func sheduleTimer(_ timer: Timer) { + timeShedule = timer.fireDate.timeIntervalSince1970 + } + + private func _test(_ kind: IOKind, _ captureTime: Double, _ delay: Double) { + print("ct \(captureTime)") + gap.localTime = zeroLocal + captureTime + delay + process(kind, time.createPacket(zeroRemote + captureTime)) + } +} + +class IOSyncTestGap : IOSyncGap { + + private var _localTime: Double = 0 + + override var localTime: Double { + get { + return _localTime + } + set { + _localTime = newValue + } + } +} + +class IOSyncTestTime : IOTimeProtocol { + + static let kTime: Int = 0 + + func createPacket(_ time: Double) -> [Int : NSData] { + var result = [Int : NSData]() + self.time(&result, time) + return result + } + + func time(_ data: [Int : NSData]) -> Double { + var result: Double = 0 + memcpy(&result, data[IOSyncTestTime.kTime]!.bytes, MemoryLayout.size) + return result + } + + func time(_ data: inout [Int : NSData], _ time_: Double) { + var time = time_ + data[IOSyncTestTime.kTime] = NSData(bytes: &time, length: MemoryLayout.size) + } +} diff --git a/apple/Common/IO/Sync.swift b/apple/Common/IO/Sync.swift deleted file mode 100644 index e721e4e..0000000 --- a/apple/Common/IO/Sync.swift +++ /dev/null @@ -1,148 +0,0 @@ - -import Foundation - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// IOSyncBus -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class IOSyncBus : IODataProtocol { - - private let kind: IOKind - private let sync: IOSync - - init(_ kind: IOKind, _ sync: IOSync) { - self.kind = kind - self.sync = sync - } - - func process(_ data: [Int : NSData]) { - sync.process(kind, data) - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// IOSync -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class IOSync { - - private struct _TimerItem { - - let id: UInt64 - let kind: IOKind - let time: Double - let data: [Int : NSData] - - init(_ id: UInt64, _ kind: IOKind, _ time: Double, _ data: [Int : NSData]) { - self.id = id - self.kind = kind - self.time = time - self.data = data - } - } - - private struct _QueueItem { - let timer: Timer - let data: _TimerItem - - init(_ timer: Timer, _ data: _TimerItem) { - self.timer = timer - self.data = data - } - } - - private var id: UInt64 = 0 - private var nextID: UInt64 { get { id += 1; return id } } - - private var output = [IOKind: IODataProtocol]() - private var timing = [IOKind: IOTimeProtocol]() - - private var localZero: Date? - private var remoteZero: Double? - private var gap: Double? - - private var thread: Thread? - private var runLoop: RunLoop? - private var queue = [UInt64: _QueueItem]() - - func add(_ kind: IOKind, _ time: IOTimeProtocol, _ output: IODataProtocol) { - self.output[kind] = output - self.timing[kind] = time - } - - func process(_ kind: IOKind, _ data: [Int : NSData]) { - - if localZero == nil { - _setup(kind, data) - } - else { - _enqueue(kind, data) - } - } - - private func _gap(_ remoteTime: Double, _ localTime: Date) -> Double { - return (remoteTime - remoteZero!) - localTime.timeIntervalSince(localZero!) - self.gap! - } - - private func _setup(_ kind: IOKind, _ data: [Int : NSData]) { - localZero = Date() - remoteZero = timing[kind]!.time(data) - gap = 0 - - thread = Thread(target: self, selector: #selector(_thread), object: nil) - thread!.start() - - output[kind]!.process(data) - } - - private func _enqueue(_ kind: IOKind, _ data: [Int : NSData]) { - let id = nextID - let remoteTime = timing[kind]!.time(data) - let localTime = Date() - let timerItem = _TimerItem(id, kind, timing[kind]!.time(data), data) - let gap = _gap(remoteTime, localTime) - - if gap < 0 { - let timer = Timer(fireAt: localTime.addingTimeInterval(-gap), - interval: 0, - target: self, - selector: #selector(_process(timer:)), - userInfo: id, - repeats: false) - - runLoop!.add(timer, forMode: .defaultRunLoopMode) - queue[id] = _QueueItem(timer, timerItem) - - logIO("sheduling \(kind) data with gap \(gap)") - } - else { - self.gap! += gap - logIO("belated \(kind) data with gap \(gap)") - logIO("changed gap to \(self.gap!)") - - _process(timerItem) - } - } - - @objc func _thread() { - runLoop = RunLoop.current - while true { - RunLoop.current.run() - } - } - - @objc private func _process(timer: Timer) { - let id = timer.userInfo as! UInt64 - - _process(queue[id]!.data) - queue.removeValue(forKey: id) - } - - private func _process(_ x: _TimerItem) { - print("process data \(x.kind) data with id \(x.id)") - - AV.shared.avOutputQueue.async { - self.output[x.kind]?.process(x.data) - } - } -} diff --git a/apple/Common/IO/Video.swift b/apple/Common/IO/Video.swift index 8dce27b..3d8a42a 100644 --- a/apple/Common/IO/Video.swift +++ b/apple/Common/IO/Video.swift @@ -124,6 +124,8 @@ class VideoTimeDeserializer : IOTimeProtocol { self.packetKey = packetKey } + static let Size = UInt32(MemoryLayout.size) + func videoTime(_ packets: [Int: NSData]) -> CMSampleTimingInfo { return CMSampleTimingInfo(packets[packetKey]!) } diff --git a/apple/Common/IO/VideoOutput.swift b/apple/Common/IO/VideoOutput.swift index 5c38d43..9f8c8f0 100644 --- a/apple/Common/IO/VideoOutput.swift +++ b/apple/Common/IO/VideoOutput.swift @@ -40,12 +40,14 @@ class VideoOutput : VideoOutputProtocol { format = dataFormat - if self.layer.isReadyForMoreMediaData && self.layer.status != .failed { - self.layer.enqueue(data) - } - else { - self.printStatus() - self.layer.flush() + dispatch_sync_on_main { + if self.layer.isReadyForMoreMediaData && self.layer.status != .failed { + self.layer.enqueue(data) + } + else { + self.printStatus() + self.layer.flush() + } } } diff --git a/apple/Common/Network/NetworkAAC.swift b/apple/Common/Network/NetworkAAC.swift index 530de14..d920973 100644 --- a/apple/Common/Network/NetworkAAC.swift +++ b/apple/Common/Network/NetworkAAC.swift @@ -72,6 +72,7 @@ class NetworkAACSerializer : AudioOutputProtocol { // output output?.process([AACPart.NetworkPacket.rawValue: NSData(bytes: result, length: size)]) + result.deallocate(capacity: size) } } diff --git a/apple/Common/Network/NetworkH264.swift b/apple/Common/Network/NetworkH264.swift index 1b42906..8d894fe 100644 --- a/apple/Common/Network/NetworkH264.swift +++ b/apple/Common/Network/NetworkH264.swift @@ -29,9 +29,11 @@ class NetworkH264Serializer : IODataProtocol { ? data[part.rawValue]!.length : 0) - result.append(&size, length: MemoryLayout.size) + if part != H264Part.Time { + result.append(&size, length: MemoryLayout.size) + } - if (size != 0) { + if size != 0 { result.append(data[part.rawValue]! as Data) } } @@ -55,34 +57,40 @@ class NetworkH264Deserializer : IODataProtocol { let packet = data[H264Part.NetworkPacket.rawValue]! var result = [Int: NSData]() - if let part = _process(data: packet, shift: &shift) { + if let part = _process(data: packet, knownSize: VideoTimeDeserializer.Size, shift: &shift) { result[H264Part.Time.rawValue] = part } - if let part = _process(data: packet, shift: &shift) { + if let part = _process(data: packet, knownSize: 0, shift: &shift) { result[H264Part.SPS.rawValue] = part } - if let part = _process(data: packet, shift: &shift) { + if let part = _process(data: packet, knownSize: 0, shift: &shift) { result[H264Part.PPS.rawValue] = part } - if let part = _process(data: packet, shift: &shift) { + if let part = _process(data: packet, knownSize: 0, shift: &shift) { result[H264Part.Data.rawValue] = part } next?.process(result) } - func _process(data: NSData, shift: inout Int) -> NSData? { + func _process(data: NSData, knownSize: UInt32, shift: inout Int) -> NSData? { var size: UInt32 = 0 - - data.getBytes(&size, range: NSRange(location: shift, length: MemoryLayout.size)) - shift += MemoryLayout.size - - if (size == 0) { - return nil + + if knownSize == 0 { + data.getBytes(&size, range: NSRange(location: shift, length: MemoryLayout.size)) + shift += MemoryLayout.size + + if (size == 0) { + assert(false) + return nil + } + } + else { + size = knownSize } let result = NSData(bytes: data.bytes.advanced(by: shift), length: Int(size)) diff --git a/apple/Common/Utils/AudioToolbox.swift b/apple/Common/Utils/AudioToolbox.swift new file mode 100644 index 0000000..153c29d --- /dev/null +++ b/apple/Common/Utils/AudioToolbox.swift @@ -0,0 +1,153 @@ +// +// AudioToolbox.swift +// Chat +// +// Created by Ivan Khvorostinin on 14/06/2017. +// Copyright © 2017 ys1382. All rights reserved. +// + +import AudioToolbox + +class AppleAudioUnit { + + let unit: AudioUnit + + init(_ unit: AudioUnit) { + self.unit = unit + } + + convenience init(_ type: OSType, _ subtype: OSType) throws { + var componentDescription = AudioComponentDescription() + componentDescription.componentType = type + componentDescription.componentSubType = subtype + componentDescription.componentManufacturer = kAudioUnitManufacturer_Apple + componentDescription.componentFlags = 0 + componentDescription.componentFlagsMask = 0 + + let component = AudioComponentFindNext(nil, &componentDescription); + var instance: AudioUnit? + + try checkStatus(AudioComponentInstanceNew(component!, + &instance), + "AudioComponentInstanceNew") + + self.init(instance!) + } + + func initialize() throws { + try checkStatus(AudioUnitInitialize(unit), + "AudioUnitInitialize") + } + + func start() throws { + try checkStatus(AudioOutputUnitStart(unit), + "AudioOutputUnitStart") + } + + func stop() throws { + try checkStatus(AudioOutputUnitStop(unit), + "AudioOutputUnitStop") + } + + func getIOEnabled(_ scope: AudioUnitScope, _ bus: Int) throws -> Bool { + var size: UInt32 = UInt32(MemoryLayout.size) + var res: UInt32 = 0 + try checkStatus(AudioUnitGetProperty(unit, + kAudioOutputUnitProperty_EnableIO, + scope, + UInt32(bus), + &res, + &size), + "AudioUnitGetProperty: kAudioOutputUnitProperty_EnableIO") + return res == 1 + + } + + func setIOEnabled(_ scope: AudioUnitScope, _ bus: Int, _ value: Bool) throws { + var valueCopy = value + #if os(iOS) + try checkStatus(AudioUnitSetProperty(unit, + kAudioOutputUnitProperty_EnableIO, + scope, + UInt32(bus), + &valueCopy, + UInt32(MemoryLayout.size)), + "AudioUnitSetProperty: kAudioOutputUnitProperty_EnableIO") + #endif + } + + func getFormat(_ scope: AudioUnitScope, _ bus: Int) throws -> AudioStreamBasicDescription { + var size: UInt32 = UInt32(MemoryLayout.size) + var res = AudioStreamBasicDescription() + try checkStatus(AudioUnitGetProperty(unit, + kAudioUnitProperty_StreamFormat, + scope, + UInt32(bus), + &res, + &size), + "AudioUnitGetProperty: kAudioUnitProperty_StreamFormat") + return res + + } + + func getFormat(_ scope: AudioUnitScope, _ bus: Int, _ outFormat: inout AudioStreamBasicDescription?) throws { + outFormat = try getFormat(scope, bus) + } + + func setFormat(_ scope: AudioUnitScope, _ bus: Int, _ value: AudioStreamBasicDescription) throws { + var valueCopy = value + try checkStatus(AudioUnitSetProperty(unit, + kAudioUnitProperty_StreamFormat, + scope, + UInt32(bus), + &valueCopy, + UInt32(MemoryLayout.size)), + "AudioUnitSetProperty: kAudioUnitProperty_StreamFormat") + } + + func setCallback(_ scope: AudioUnitScope, _ bus: Int, _ value: inout AURenderCallbackStruct) throws { + try checkStatus(AudioUnitSetProperty(unit, + kAudioOutputUnitProperty_SetInputCallback, + scope, + UInt32(bus), + &value, + UInt32(MemoryLayout.size)), + "AudioUnitSetProperty: kAudioOutputUnitProperty_SetInputCallback") + } + + func setRenderer(_ scope: AudioUnitScope, _ bus: Int, _ value: inout AURenderCallbackStruct) throws { + try checkStatus(AudioUnitSetProperty(unit, + kAudioUnitProperty_SetRenderCallback, + scope, + UInt32(bus), + &value, + UInt32(MemoryLayout.size)), + "AudioUnitSetProperty: kAudioUnitProperty_SetRenderCallback") + } + + func getMatrixVolume(_ scope: AudioUnitScope, _ bus: Int) throws -> Double { + var res: Float32 = 0 + var size: UInt32 = UInt32(MemoryLayout.size) + try checkStatus(AudioUnitGetProperty(unit, + kMatrixMixerParam_Volume, + scope, + UInt32(bus), + &res, + &size), + "AudioUnitGetProperty: kAudioUnitProperty_StreamFormat") + return Double(res) + } + + func getMultiChannelVolume(_ scope: AudioUnitScope, _ bus: Int) throws -> Double { + var res: Float32 = 0 + var size: UInt32 = UInt32(MemoryLayout.size) + try checkStatus(AudioUnitGetProperty(unit, + kMultiChannelMixerParam_Volume, + scope, + UInt32(bus), + &res, + &size), + "AudioUnitGetProperty: kAudioUnitProperty_StreamFormat") + return Double(res) + } +} diff --git a/apple/Common/Utils/CoreAudio.swift b/apple/Common/Utils/CoreAudio.swift index 3eed4d5..365f6f3 100644 --- a/apple/Common/Utils/CoreAudio.swift +++ b/apple/Common/Utils/CoreAudio.swift @@ -2,32 +2,55 @@ import CoreAudio extension AudioStreamBasicDescription { + + typealias Factory = () throws -> AudioStreamBasicDescription? - static func CreateInput(_ formatID: UInt32, - _ sampleRate: Double, - _ channelCount: UInt32) -> AudioStreamBasicDescription { + // constant bit rate + static func CreateCBR(_ formatID: UInt32, + _ sampleRate: Double, + _ channelCount: UInt32) -> AudioStreamBasicDescription { var result = AudioStreamBasicDescription() result.mSampleRate = sampleRate; result.mChannelsPerFrame = channelCount; result.mFormatID = formatID; - if (formatID == kAudioFormatLinearPCM) - { - // if we want pcm, default to signed 16-bit little-endian - result.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - result.mBitsPerChannel = 16; - result.mBytesPerFrame = (result.mBitsPerChannel / 8) * result.mChannelsPerFrame; - result.mBytesPerPacket = result.mBytesPerFrame - result.mFramesPerPacket = 1; - } + // if we want pcm, default to signed 16-bit little-endian + result.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked + result.mBitsPerChannel = 16; + result.mBytesPerFrame = (result.mBitsPerChannel / 8) * result.mChannelsPerFrame + result.mBytesPerPacket = result.mBytesPerFrame + result.mFramesPerPacket = 1; + + return result + } + + // constant bit rate + static func CreateCBR(_ format: AudioFormat) -> AudioStreamBasicDescription { + var result = AudioStreamBasicDescription.CreateCBR(format.formatID, format.sampleRate, format.channelCount) + + result.mFormatFlags = format.flags + result.mFramesPerPacket = format.framesPerPacket + + return result + } + + // variable bit rate + static func CreateVBR(_ formatID: UInt32, + _ sampleRate: Double, + _ channelCount: UInt32) -> AudioStreamBasicDescription { + var result = AudioStreamBasicDescription() + + result.mSampleRate = sampleRate + result.mChannelsPerFrame = channelCount + result.mFormatID = formatID return result } - static func CreateOutput(_ format: AudioFormat, - _ formatID: UInt32) -> AudioStreamBasicDescription { - var result = AudioStreamBasicDescription.CreateInput(formatID, format.sampleRate, format.channelCount) + // variable bit rate + static func CreateVBR(_ format: AudioFormat) -> AudioStreamBasicDescription { + var result = AudioStreamBasicDescription.CreateVBR(format.formatID, format.sampleRate, format.channelCount) result.mFormatFlags = format.flags result.mFramesPerPacket = format.framesPerPacket @@ -39,10 +62,10 @@ extension AudioStreamBasicDescription { extension AudioTimeStamp { func seconds() -> Double { - return Double(mHostTime) / 1000000000.0 + return mach_absolute_seconds(mHostTime) } mutating func seconds(_ x: Double) { - mHostTime = UInt64(x * 1000000000.0) + mHostTime = mach_absolute_time(seconds: x) } } diff --git a/apple/Common/Utils/Defs.swift b/apple/Common/Utils/Defs.swift index bd3bcc6..55b339c 100644 --- a/apple/Common/Utils/Defs.swift +++ b/apple/Common/Utils/Defs.swift @@ -22,3 +22,4 @@ import Foundation //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// typealias FuncVV = () -> Void +typealias FuncVVT = () throws -> Void diff --git a/apple/Common/Utils/System.swift b/apple/Common/Utils/System.swift new file mode 100644 index 0000000..119b3dc --- /dev/null +++ b/apple/Common/Utils/System.swift @@ -0,0 +1,57 @@ + +import Foundation + +fileprivate class HostTimeInfo { + static let shared = HostTimeInfo() + + let numer: UInt32 + let denom: UInt32 + + init() { + var info = mach_timebase_info_data_t() + mach_timebase_info(&info) + + numer = info.numer + denom = info.denom + } +} + +func seconds2nano(_ seconds: Double) -> Int64 { + return nano(seconds) +} + +func seconds4nano(_ nano: Int64) -> Double { + return Double(nano) / 1000000000.0 +} + +func milli(_ x: Double) -> Int64 { + return Int64(x * 1000.0) +} + +func micro(_ x: Double) -> Int64 { + return Int64(x * 1000.0 * 1000.0) +} + +func nano(_ x: Double) -> Int64 { + return Int64(x * 1000.0 * 1000.0 * 1000.0) +} + +func mach_absolute_seconds(_ machTime: UInt64) -> Double { + return + seconds4nano( + Int64(Double(machTime * UInt64(HostTimeInfo.shared.numer)) / Double(HostTimeInfo.shared.denom))) +} + +func mach_absolute_seconds() -> Double { + return mach_absolute_seconds(mach_absolute_time()) +} + +func mach_absolute_time(seconds: Double) -> UInt64 { + return + UInt64(seconds2nano( + seconds * Double(HostTimeInfo.shared.denom) / Double(HostTimeInfo.shared.numer))) +} + +func typeName(_ some: Any) -> String { + return (some is Any.Type) ? "\(some)" : "\(type(of: some))" +} diff --git a/apple/Common/Utils/Thread.swift b/apple/Common/Utils/Thread.swift index 088f7f5..dfd4ebf 100644 --- a/apple/Common/Utils/Thread.swift +++ b/apple/Common/Utils/Thread.swift @@ -12,6 +12,10 @@ class ChatThread : Thread { self.name = name } + convenience init(_ type: T) { + self.init(typeName(type)) + } + override func main() { runLoop = RunLoop.current runLoop!.add(NSMachPort(), forMode: .defaultRunLoopMode) @@ -22,6 +26,11 @@ class ChatThread : Thread { } } + override func start() { + super.start() + sync { /* wait for RunLoop initialization */ } + } + override func cancel() { running = false super.cancel() diff --git a/apple/ios/Chat.xcodeproj/project.pbxproj b/apple/ios/Chat.xcodeproj/project.pbxproj index 99b1070..c8acb64 100644 --- a/apple/ios/Chat.xcodeproj/project.pbxproj +++ b/apple/ios/Chat.xcodeproj/project.pbxproj @@ -13,14 +13,14 @@ 66C4B5946A64F209719F1358 /* Pods_Chat.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0ACBFB437F9424A73A5B0830 /* Pods_Chat.framework */; }; 821F4CC71EDEB03500FA923D /* Dispatch.swift in Sources */ = {isa = PBXBuildFile; fileRef = 821F4CC61EDEB03500FA923D /* Dispatch.swift */; }; 82203E6D1EE6B44F005AEC49 /* VideoOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82203E6C1EE6B44F005AEC49 /* VideoOutput.swift */; }; + 822775301EF126390083D9DB /* IOSync.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8227752F1EF126390083D9DB /* IOSync.swift */; }; + 822775321EF126AC0083D9DB /* System.swift in Sources */ = {isa = PBXBuildFile; fileRef = 822775311EF126AC0083D9DB /* System.swift */; }; 825E1AF11EEA7FFB00C26112 /* SettingsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 825E1AF01EEA7FFB00C26112 /* SettingsViewController.swift */; }; 825E1AF31EEB179E00C26112 /* Thread.swift in Sources */ = {isa = PBXBuildFile; fileRef = 825E1AF21EEB179E00C26112 /* Thread.swift */; }; 8260DDE51EE47918004416FF /* Fabric.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8260DDE31EE47918004416FF /* Fabric.framework */; }; 8260DDE61EE47918004416FF /* Crashlytics.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8260DDE41EE47918004416FF /* Crashlytics.framework */; }; 8260DDE81EE55C59004416FF /* !Video.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8260DDE71EE55C59004416FF /* !Video.swift */; }; 8260DDEA1EE55E04004416FF /* !AVFoundation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8260DDE91EE55E04004416FF /* !AVFoundation.swift */; }; - 8275DDA61EE68DB800BF422B /* Sync.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8275DDA51EE68DB800BF422B /* Sync.swift */; }; - 827C52071EE01E4200340D56 /* !Audio.swift in Sources */ = {isa = PBXBuildFile; fileRef = 827C52061EE01E4200340D56 /* !Audio.swift */; }; 827C52091EE01EB800340D56 /* CoreAudio.swift in Sources */ = {isa = PBXBuildFile; fileRef = 827C52081EE01EB800340D56 /* CoreAudio.swift */; }; 8286E3101EDC1AFF006788B2 /* Backend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8286E2FE1EDC1AFF006788B2 /* Backend.swift */; }; 8286E3111EDC1AFF006788B2 /* IO.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8286E3001EDC1AFF006788B2 /* IO.swift */; }; @@ -48,6 +48,8 @@ 829A0EC11EDD936C00FEC353 /* AV.swift in Sources */ = {isa = PBXBuildFile; fileRef = 829A0EC01EDD936C00FEC353 /* AV.swift */; }; 82A68C0C1EDBDE0500049AD9 /* MediaViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82A68C0B1EDBDE0500049AD9 /* MediaViewController.swift */; }; 82DECAAB1EEB1A25008199AC /* Defs.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82DECAAA1EEB1A25008199AC /* Defs.swift */; }; + 82EA1AB81EF1C3E5004E8360 /* AudioDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82EA1AB71EF1C3E5004E8360 /* AudioDecoder.swift */; }; + 82EA1ABA1EF1C3FC004E8360 /* AudioToolbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82EA1AB91EF1C3FC004E8360 /* AudioToolbox.swift */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -59,14 +61,14 @@ 1AB062191E8843E600A841D0 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 821F4CC61EDEB03500FA923D /* Dispatch.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Dispatch.swift; sourceTree = ""; }; 82203E6C1EE6B44F005AEC49 /* VideoOutput.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VideoOutput.swift; sourceTree = ""; }; + 8227752F1EF126390083D9DB /* IOSync.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOSync.swift; sourceTree = ""; }; + 822775311EF126AC0083D9DB /* System.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = System.swift; sourceTree = ""; }; 825E1AF01EEA7FFB00C26112 /* SettingsViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SettingsViewController.swift; sourceTree = ""; }; 825E1AF21EEB179E00C26112 /* Thread.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Thread.swift; sourceTree = ""; }; 8260DDE31EE47918004416FF /* Fabric.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = Fabric.framework; sourceTree = ""; }; 8260DDE41EE47918004416FF /* Crashlytics.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = Crashlytics.framework; sourceTree = ""; }; 8260DDE71EE55C59004416FF /* !Video.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "!Video.swift"; sourceTree = ""; }; 8260DDE91EE55E04004416FF /* !AVFoundation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "!AVFoundation.swift"; sourceTree = ""; }; - 8275DDA51EE68DB800BF422B /* Sync.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Sync.swift; sourceTree = ""; }; - 827C52061EE01E4200340D56 /* !Audio.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "!Audio.swift"; sourceTree = ""; }; 827C52081EE01EB800340D56 /* CoreAudio.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CoreAudio.swift; sourceTree = ""; }; 8286E2FE1EDC1AFF006788B2 /* Backend.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Backend.swift; path = ../Common/Backend.swift; sourceTree = ""; }; 8286E3001EDC1AFF006788B2 /* IO.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IO.swift; sourceTree = ""; }; @@ -95,6 +97,8 @@ 82A68C0B1EDBDE0500049AD9 /* MediaViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaViewController.swift; sourceTree = ""; }; 82A68C451EDC18D200049AD9 /* libChatLib.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libChatLib.a; path = "../common/build/Debug-iphoneos/libChatLib.a"; sourceTree = ""; }; 82DECAAA1EEB1A25008199AC /* Defs.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Defs.swift; sourceTree = ""; }; + 82EA1AB71EF1C3E5004E8360 /* AudioDecoder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioDecoder.swift; sourceTree = ""; }; + 82EA1AB91EF1C3FC004E8360 /* AudioToolbox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioToolbox.swift; sourceTree = ""; }; C5A48039D30C2E9ECC041BE6 /* Pods-Chat.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Chat.release.xcconfig"; path = "Pods/Target Support Files/Pods-Chat/Pods-Chat.release.xcconfig"; sourceTree = ""; }; F1950861D287C0179BA10F15 /* Pods-Chat.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Chat.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Chat/Pods-Chat.debug.xcconfig"; sourceTree = ""; }; /* End PBXFileReference section */ @@ -169,7 +173,6 @@ 827C52051EE01E3800340D56 /* Utils */ = { isa = PBXGroup; children = ( - 827C52061EE01E4200340D56 /* !Audio.swift */, 8260DDE71EE55C59004416FF /* !Video.swift */, 8260DDE91EE55E04004416FF /* !AVFoundation.swift */, ); @@ -195,15 +198,16 @@ isa = PBXGroup; children = ( 8286E3001EDC1AFF006788B2 /* IO.swift */, + 8227752F1EF126390083D9DB /* IOSync.swift */, 829A0EB21EDD52D400FEC353 /* Audio.swift */, 829A0EB41EDD52DE00FEC353 /* Video.swift */, 8286E3011EDC1AFF006788B2 /* AudioInput.swift */, 8286E3021EDC1AFF006788B2 /* AudioOutput.swift */, + 82EA1AB71EF1C3E5004E8360 /* AudioDecoder.swift */, 8286E3061EDC1AFF006788B2 /* VideoInput.swift */, 82203E6C1EE6B44F005AEC49 /* VideoOutput.swift */, 8286E3041EDC1AFF006788B2 /* VideoDecoderH264.swift */, 8286E3051EDC1AFF006788B2 /* VideoEncoderH264.swift */, - 8275DDA51EE68DB800BF422B /* Sync.swift */, ); name = IO; path = ../Common/IO; @@ -223,11 +227,13 @@ 8286E30B1EDC1AFF006788B2 /* Utils */ = { isa = PBXGroup; children = ( + 822775311EF126AC0083D9DB /* System.swift */, 82DECAAA1EEB1A25008199AC /* Defs.swift */, 829A0EBC1EDD6EFB00FEC353 /* Foundation.swift */, 8286E30C1EDC1AFF006788B2 /* AVFoundation.swift */, 8286E30D1EDC1AFF006788B2 /* CoreMedia.swift */, 827C52081EE01EB800340D56 /* CoreAudio.swift */, + 82EA1AB91EF1C3FC004E8360 /* AudioToolbox.swift */, 8286E30E1EDC1AFF006788B2 /* Log.swift */, 829A0EAE1EDD2C2B00FEC353 /* UI.swift */, 821F4CC61EDEB03500FA923D /* Dispatch.swift */, @@ -389,6 +395,8 @@ 828E30271EEC0FAE00597EB8 /* Application.swift in Sources */, 8286E3161EDC1AFF006788B2 /* VideoEncoderH264.swift in Sources */, 8286E3201EDC1B1D006788B2 /* NetworkH264.swift in Sources */, + 82EA1AB81EF1C3E5004E8360 /* AudioDecoder.swift in Sources */, + 822775321EF126AC0083D9DB /* System.swift in Sources */, 825E1AF11EEA7FFB00C26112 /* SettingsViewController.swift in Sources */, 8286E31A1EDC1AFF006788B2 /* NetworkAAC.swift in Sources */, 8286E3131EDC1AFF006788B2 /* AudioOutput.swift in Sources */, @@ -402,16 +410,17 @@ 8286E3181EDC1AFF006788B2 /* Model.swift in Sources */, 8286E31C1EDC1AFF006788B2 /* CoreMedia.swift in Sources */, 829A0EB51EDD52DE00FEC353 /* Video.swift in Sources */, + 822775301EF126390083D9DB /* IOSync.swift in Sources */, 829A0EC11EDD936C00FEC353 /* AV.swift in Sources */, 8286E31D1EDC1AFF006788B2 /* Log.swift in Sources */, 8286E3171EDC1AFF006788B2 /* VideoInput.swift in Sources */, 82A68C0C1EDBDE0500049AD9 /* MediaViewController.swift in Sources */, + 82EA1ABA1EF1C3FC004E8360 /* AudioToolbox.swift in Sources */, 829A0E791EDC1CE200FEC353 /* SplitViewController.swift in Sources */, 829A0EBD1EDD6EFB00FEC353 /* Foundation.swift in Sources */, 8286E3121EDC1AFF006788B2 /* AudioInput.swift in Sources */, 82DECAAB1EEB1A25008199AC /* Defs.swift in Sources */, 82203E6D1EE6B44F005AEC49 /* VideoOutput.swift in Sources */, - 827C52071EE01E4200340D56 /* !Audio.swift in Sources */, 8260DDE81EE55C59004416FF /* !Video.swift in Sources */, 827C52091EE01EB800340D56 /* CoreAudio.swift in Sources */, 8260DDEA1EE55E04004416FF /* !AVFoundation.swift in Sources */, @@ -419,7 +428,6 @@ 829A0EAF1EDD2C2B00FEC353 /* UI.swift in Sources */, 829A0EB31EDD52D400FEC353 /* Audio.swift in Sources */, 8286E3111EDC1AFF006788B2 /* IO.swift in Sources */, - 8275DDA61EE68DB800BF422B /* Sync.swift in Sources */, 1AB0620C1E8843E500A841D0 /* AppDelegate.swift in Sources */, 821F4CC71EDEB03500FA923D /* Dispatch.swift in Sources */, ); diff --git a/apple/ios/Chat/AppDelegate.swift b/apple/ios/Chat/AppDelegate.swift index 2d93c90..67f3963 100644 --- a/apple/ios/Chat/AppDelegate.swift +++ b/apple/ios/Chat/AppDelegate.swift @@ -1,4 +1,5 @@ import UIKit +import AVFoundation @UIApplicationMain class AppDelegate: Application, UIApplicationDelegate, UISplitViewControllerDelegate { @@ -22,8 +23,16 @@ class AppDelegate: Application, UIApplicationDelegate, UISplitViewControllerDele // Audio - AV.shared.setupDefaultNetworkInputAudio(ChatAudioSession()) - + do { + try AVAudioSession.sharedInstance().setCategory(AVAudioSessionCategoryPlayAndRecord) + try AVAudioSession.sharedInstance().setActive(true) + + AV.shared.setupDefaultNetworkAudioOutput(nil) + } + catch { + logIOError(error) + } + // done return true diff --git a/apple/ios/Chat/UI/MediaViewController.swift b/apple/ios/Chat/UI/MediaViewController.swift index 6b3adc1..d8264f4 100644 --- a/apple/ios/Chat/UI/MediaViewController.swift +++ b/apple/ios/Chat/UI/MediaViewController.swift @@ -50,14 +50,13 @@ class MediaViewController : UIViewController { videoSession!, video) - audio = - ChatAudioSession( - audio) - - audio = IOSessionAsyncDispatcher(AV.shared.audioCaptureQueue, audio!) video = VideoSessionAsyncDispatcher(AV.shared.videoCaptureQueue, video!) } + if audio != nil { + audio = IOSessionAsyncDispatcher(AV.shared.audioCaptureQueue, audio!) + } + try AV.shared.startInput(create([audio, video])); } catch { @@ -77,13 +76,17 @@ class MediaViewController : UIViewController { // setup video output - videoSessionStart = { (_ id: IOID, _) in + videoSessionStart = { (_ id: IOID, _) throws in self.sessionID = id.sid - return AV.shared.defaultNetworkInputVideo(id, VideoOutput(self.networkView.sampleLayer)) + + let result = AV.shared.defaultNetworkVideoOutput(id, VideoOutput(self.networkView.sampleLayer)) + try AV.shared.defaultIOSync(id.gid).start() + return result } - videoSessionStop = { (_) in + videoSessionStop = { (_ id: IOID) in self.networkView.sampleLayer.flushAndRemoveImage() + try! AV.shared.defaultIOSync(id.gid).stop() } } diff --git a/apple/ios/Chat/UI/SplitViewController.swift b/apple/ios/Chat/UI/SplitViewController.swift index fcd51a6..4d94e80 100644 --- a/apple/ios/Chat/UI/SplitViewController.swift +++ b/apple/ios/Chat/UI/SplitViewController.swift @@ -38,7 +38,7 @@ class SplitViewController : UISplitViewController { override func viewDidLoad() { Backend.shared.videoSessionStart = { (_ id: IOID, _ format: VideoFormat) throws -> IODataProtocol? in - assert_main_queue() + assert_main() if self.detailViewController == nil { let detailsID = String(describing: DetailViewController.self) diff --git a/apple/ios/Chat/Utils/!Audio.swift b/apple/ios/Chat/Utils/!Audio.swift deleted file mode 100644 index bc94243..0000000 --- a/apple/ios/Chat/Utils/!Audio.swift +++ /dev/null @@ -1,33 +0,0 @@ - -import AVFoundation - -class ChatAudioSession : IOSessionProtocol { - - let next: IOSessionProtocol? - - convenience init() { - self.init(nil) - } - - init(_ next: IOSessionProtocol?) { - self.next = next - } - - func start() throws { - try AVAudioSession.sharedInstance().setCategory(AVAudioSessionCategoryPlayAndRecord) - try AVAudioSession.sharedInstance().setActive(true) - - try next?.start() - } - - func stop() { - next?.stop() - -// do { -// try AVAudioSession.sharedInstance().setActive(false) -// } -// catch { -// logIOError(error) -// } - } -} diff --git a/apple/osx/Chat.xcodeproj/project.pbxproj b/apple/osx/Chat.xcodeproj/project.pbxproj index 94dc22c..900b928 100644 --- a/apple/osx/Chat.xcodeproj/project.pbxproj +++ b/apple/osx/Chat.xcodeproj/project.pbxproj @@ -10,6 +10,7 @@ 1ADE89E51EA1CE0E00E41212 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1ADE89E41EA1CE0E00E41212 /* AppDelegate.swift */; }; 1ADE89E91EA1CE0E00E41212 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 1ADE89E81EA1CE0E00E41212 /* Assets.xcassets */; }; 1ADE89EC1EA1CE0E00E41212 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 1ADE89EA1EA1CE0E00E41212 /* Main.storyboard */; }; + 821D05181EF1BD070020E1BA /* AudioDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 821D05171EF1BD070020E1BA /* AudioDecoder.swift */; }; 821F4CC51EDE6E6500FA923D /* SplitViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 821F4CC41EDE6E6500FA923D /* SplitViewController.swift */; }; 821F4CC91EDEB3A100FA923D /* Dispatch.swift in Sources */ = {isa = PBXBuildFile; fileRef = 821F4CC81EDEB3A100FA923D /* Dispatch.swift */; }; 821F4CCD1EE00D2800FA923D /* CoreAudio.swift in Sources */ = {isa = PBXBuildFile; fileRef = 821F4CCC1EE00D2800FA923D /* CoreAudio.swift */; }; @@ -17,7 +18,8 @@ 82203E711EE6F0E2005AEC49 /* Fabric.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 82203E6F1EE6F0E2005AEC49 /* Fabric.framework */; }; 82203E721EE6F0E2005AEC49 /* Crashlytics.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 82203E701EE6F0E2005AEC49 /* Crashlytics.framework */; }; 8221F0901EE15F7400FE253A /* ChatViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8221F08F1EE15F7400FE253A /* ChatViewController.swift */; }; - 8275DDA41EE66E9700BF422B /* Sync.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8275DDA31EE66E9700BF422B /* Sync.swift */; }; + 822775341EF140640083D9DB /* AudioToolbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = 822775331EF140640083D9DB /* AudioToolbox.swift */; }; + 8275DDA41EE66E9700BF422B /* IOSync.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8275DDA31EE66E9700BF422B /* IOSync.swift */; }; 828E30291EEC176600597EB8 /* Application.swift in Sources */ = {isa = PBXBuildFile; fileRef = 828E30281EEC176600597EB8 /* Application.swift */; }; 829A0E8D1EDC1DB800FEC353 /* Backend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 829A0E7B1EDC1DB800FEC353 /* Backend.swift */; }; 829A0E8E1EDC1DB800FEC353 /* AudioInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = 829A0E7D1EDC1DB800FEC353 /* AudioInput.swift */; }; @@ -42,6 +44,7 @@ 829A0EB91EDD54C800FEC353 /* Video.swift in Sources */ = {isa = PBXBuildFile; fileRef = 829A0EB71EDD54C800FEC353 /* Video.swift */; }; 829A0EBB1EDD6EED00FEC353 /* Foundation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 829A0EBA1EDD6EED00FEC353 /* Foundation.swift */; }; 829A0EBF1EDD784E00FEC353 /* AV.swift in Sources */ = {isa = PBXBuildFile; fileRef = 829A0EBE1EDD784E00FEC353 /* AV.swift */; }; + 82B8A8811EEEAFE1006002FD /* System.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82B8A8801EEEAFE1006002FD /* System.swift */; }; 82DECAAE1EEB28BF008199AC /* Defs.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82DECAAC1EEB28BF008199AC /* Defs.swift */; }; 82DECAAF1EEB28BF008199AC /* Thread.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82DECAAD1EEB28BF008199AC /* Thread.swift */; }; D4C11B8E12CA4AE1016CA052 /* Pods_Chat.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F3DC7F985F75CB41DAED56C7 /* Pods_Chat.framework */; }; @@ -54,6 +57,7 @@ 1ADE89EB1EA1CE0E00E41212 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 1ADE89ED1EA1CE0E00E41212 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 64A385B4166E340653D3B04C /* Pods-Chat.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Chat.release.xcconfig"; path = "Pods/Target Support Files/Pods-Chat/Pods-Chat.release.xcconfig"; sourceTree = ""; }; + 821D05171EF1BD070020E1BA /* AudioDecoder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioDecoder.swift; sourceTree = ""; }; 821F4CC41EDE6E6500FA923D /* SplitViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SplitViewController.swift; sourceTree = ""; }; 821F4CC81EDEB3A100FA923D /* Dispatch.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Dispatch.swift; sourceTree = ""; }; 821F4CCC1EE00D2800FA923D /* CoreAudio.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CoreAudio.swift; sourceTree = ""; }; @@ -61,7 +65,8 @@ 82203E6F1EE6F0E2005AEC49 /* Fabric.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = Fabric.framework; sourceTree = ""; }; 82203E701EE6F0E2005AEC49 /* Crashlytics.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = Crashlytics.framework; sourceTree = ""; }; 8221F08F1EE15F7400FE253A /* ChatViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatViewController.swift; sourceTree = ""; }; - 8275DDA31EE66E9700BF422B /* Sync.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Sync.swift; sourceTree = ""; }; + 822775331EF140640083D9DB /* AudioToolbox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioToolbox.swift; sourceTree = ""; }; + 8275DDA31EE66E9700BF422B /* IOSync.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOSync.swift; sourceTree = ""; }; 828E30281EEC176600597EB8 /* Application.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Application.swift; path = ../Common/Application.swift; sourceTree = ""; }; 829A0E7B1EDC1DB800FEC353 /* Backend.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Backend.swift; path = ../Common/Backend.swift; sourceTree = ""; }; 829A0E7D1EDC1DB800FEC353 /* AudioInput.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioInput.swift; sourceTree = ""; }; @@ -86,6 +91,7 @@ 829A0EB71EDD54C800FEC353 /* Video.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Video.swift; sourceTree = ""; }; 829A0EBA1EDD6EED00FEC353 /* Foundation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Foundation.swift; sourceTree = ""; }; 829A0EBE1EDD784E00FEC353 /* AV.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = AV.swift; path = ../Common/AV.swift; sourceTree = ""; }; + 82B8A8801EEEAFE1006002FD /* System.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = System.swift; sourceTree = ""; }; 82DECAAC1EEB28BF008199AC /* Defs.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Defs.swift; sourceTree = ""; }; 82DECAAD1EEB28BF008199AC /* Thread.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Thread.swift; sourceTree = ""; }; 931CB699F7145BBD7B223700 /* Pods-Chat.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Chat.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Chat/Pods-Chat.debug.xcconfig"; sourceTree = ""; }; @@ -176,15 +182,16 @@ isa = PBXGroup; children = ( 829A0E7F1EDC1DB800FEC353 /* IO.swift */, + 8275DDA31EE66E9700BF422B /* IOSync.swift */, 829A0EB61EDD54C800FEC353 /* Audio.swift */, 829A0E7D1EDC1DB800FEC353 /* AudioInput.swift */, 829A0E7E1EDC1DB800FEC353 /* AudioOutput.swift */, + 821D05171EF1BD070020E1BA /* AudioDecoder.swift */, 829A0EB71EDD54C800FEC353 /* Video.swift */, 829A0E821EDC1DB800FEC353 /* VideoInput.swift */, 82203E6A1EE6B345005AEC49 /* VideoOutput.swift */, 829A0E801EDC1DB800FEC353 /* VideoDecoderH264.swift */, 829A0E811EDC1DB800FEC353 /* VideoEncoderH264.swift */, - 8275DDA31EE66E9700BF422B /* Sync.swift */, ); name = IO; path = ../Common/IO; @@ -211,8 +218,10 @@ 829A0E891EDC1DB800FEC353 /* AVFoundation.swift */, 829A0E8A1EDC1DB800FEC353 /* CoreMedia.swift */, 821F4CCC1EE00D2800FA923D /* CoreAudio.swift */, + 822775331EF140640083D9DB /* AudioToolbox.swift */, 829A0EB01EDD2CC000FEC353 /* UI.swift */, 829A0E8B1EDC1DB800FEC353 /* Log.swift */, + 82B8A8801EEEAFE1006002FD /* System.swift */, ); name = Utils; path = ../Common/Utils; @@ -368,7 +377,7 @@ buildActionMask = 2147483647; files = ( 829A0E951EDC1DB800FEC353 /* Network.swift in Sources */, - 8275DDA41EE66E9700BF422B /* Sync.swift in Sources */, + 8275DDA41EE66E9700BF422B /* IOSync.swift in Sources */, 829A0E9A1EDC1DB800FEC353 /* Log.swift in Sources */, 829A0EBF1EDD784E00FEC353 /* AV.swift in Sources */, 829A0E911EDC1DB800FEC353 /* VideoDecoderH264.swift in Sources */, @@ -384,13 +393,16 @@ 8221F0901EE15F7400FE253A /* ChatViewController.swift in Sources */, 829A0EA51EDC209A00FEC353 /* ContactsViewController.swift in Sources */, 821F4CC51EDE6E6500FA923D /* SplitViewController.swift in Sources */, + 822775341EF140640083D9DB /* AudioToolbox.swift in Sources */, 829A0E901EDC1DB800FEC353 /* IO.swift in Sources */, 829A0E8E1EDC1DB800FEC353 /* AudioInput.swift in Sources */, 829A0E9B1EDC1DB800FEC353 /* Wire.proto.swift in Sources */, + 821D05181EF1BD070020E1BA /* AudioDecoder.swift in Sources */, 829A0EB11EDD2CC000FEC353 /* UI.swift in Sources */, 829A0E981EDC1DB800FEC353 /* AVFoundation.swift in Sources */, 829A0E8D1EDC1DB800FEC353 /* Backend.swift in Sources */, 829A0E931EDC1DB800FEC353 /* VideoInput.swift in Sources */, + 82B8A8811EEEAFE1006002FD /* System.swift in Sources */, 828E30291EEC176600597EB8 /* Application.swift in Sources */, 82DECAAE1EEB28BF008199AC /* Defs.swift in Sources */, 829A0E991EDC1DB800FEC353 /* CoreMedia.swift in Sources */, diff --git a/apple/osx/Chat/AppDelegate.swift b/apple/osx/Chat/AppDelegate.swift index 4c49e1b..3f681bb 100644 --- a/apple/osx/Chat/AppDelegate.swift +++ b/apple/osx/Chat/AppDelegate.swift @@ -36,7 +36,7 @@ class AppDelegate: Application, NSApplicationDelegate { } } - AV.shared.setupDefaultNetworkInputAudio(nil) + AV.shared.setupDefaultNetworkAudioOutput(nil) } static func ask(title: String, subtitle: String, cancelable: Bool, done:(String?)->Void) { diff --git a/apple/osx/Chat/UI/VideoViewController.swift b/apple/osx/Chat/UI/VideoViewController.swift index 19f4269..ed2f7d5 100644 --- a/apple/osx/Chat/UI/VideoViewController.swift +++ b/apple/osx/Chat/UI/VideoViewController.swift @@ -57,7 +57,9 @@ class VideoViewController: NSViewController { // setup video output Backend.shared.videoSessionStart = { (_ id: IOID, _) in - return AV.shared.defaultNetworkInputVideo(id, VideoOutput(self.network.sampleLayer)) + let result = AV.shared.defaultNetworkVideoOutput(id, VideoOutput(self.network.sampleLayer)) + try! AV.shared.defaultIOSync(id.gid).start() + return result } Backend.shared.videoSessionStop = { (_) in