| |
| |
| |
| |
| |
| |
|
|
| import CoreAudio |
| import Foundation |
| import AVFAudio |
| import OSLog |
|
|
|
|
| class AudioDataPacket { |
| let audioData: Data |
| let isVoiceActive: Bool |
| |
| init(audioData: Data, isVoiceActive: Bool) { |
| self.audioData = audioData |
| self.isVoiceActive = isVoiceActive |
| } |
| } |
|
|
| class AudioDataQueue { |
| private var queue = [AudioDataPacket]() |
| private let lock = NSLock() |
| private let capacity: Int |
| |
| init(capacity: Int = 100) { |
| self.capacity = capacity |
| } |
| |
| func push(data: Data, isVoiceActive: Bool) -> Bool { |
| lock.lock() |
| defer { lock.unlock() } |
| |
| if queue.count < capacity { |
| queue.append(AudioDataPacket(audioData: data, isVoiceActive: isVoiceActive)) |
| return true |
| } |
| return false |
| } |
| |
| func pop() -> AudioDataPacket? { |
| lock.lock() |
| defer { lock.unlock() } |
| |
| if !queue.isEmpty { |
| return queue.removeFirst() |
| } |
| return nil |
| } |
| |
| var isEmpty: Bool { |
| lock.lock() |
| defer { lock.unlock() } |
| return queue.isEmpty |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| public enum AECAudioStreamError: Error{ |
| |
| case osStatusError(status: OSStatus) |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| public class AECAudioStream { |
| |
| private(set) var audioUnit: AudioUnit? |
| |
| private(set) var graph: AUGraph? |
| |
| private(set) var streamBasicDescription: AudioStreamBasicDescription |
| |
| private let logger = Logger(subsystem: "com.0x67.echo-cancellation.AECAudioUnit", category: "AECAudioStream") |
| |
| private(set) var sampleRate: Float64 |
| |
| private(set) var streamFormat: AVAudioFormat |
| |
| private(set) var enableAutomaticEchoCancellation: Bool = false |
| |
| |
| public var rendererClosure: ((UnsafeMutablePointer<AudioBufferList>, UInt32) -> Void)? |
| |
| |
| public var enableRendererCallback: Bool = false |
| |
| private(set) var capturedFrameHandler: ((AVAudioPCMBuffer) -> Void)? |
| |
| |
| private var deviceID: AudioObjectID = 0 |
| private(set) var isVoiceActivityDetectionEnabled: Bool = false |
| private(set) var isVoiceDetected: Bool = false |
| |
| |
| public var voiceActivityHandler: ((Bool) -> Void)? |
|
|
| public func updateVoiceDetectionState(_ detected: Bool) { |
| self.isVoiceDetected = detected |
| |
| self.voiceActivityHandler?(detected) |
| |
| |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| public init(sampleRate: Float64, |
| enableRendererCallback: Bool = false, |
| rendererClosure: ((UnsafeMutablePointer<AudioBufferList>, UInt32) -> Void)? = nil) { |
| self.sampleRate = sampleRate |
| self.streamBasicDescription = Self.canonicalStreamDescription(sampleRate: sampleRate) |
| self.streamFormat = AVAudioFormat(streamDescription: &self.streamBasicDescription)! |
| self.enableRendererCallback = enableRendererCallback |
| self.rendererClosure = rendererClosure |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| public func startAudioStream(enableAEC: Bool, |
| enableRendererCallback: Bool = false, |
| rendererClosure: ((UnsafeMutablePointer<AudioBufferList>, UInt32) -> Void)? = nil) -> AsyncThrowingStream<AVAudioPCMBuffer, Error> { |
| AsyncThrowingStream<AVAudioPCMBuffer, Error> { continuation in |
| do { |
| |
| self.enableRendererCallback = enableRendererCallback |
| self.rendererClosure = rendererClosure |
| self.capturedFrameHandler = {continuation.yield($0)} |
| |
| try createAUGraphForAudioUnit() |
| try configureAudioUnit() |
| try toggleAudioCancellation(enable: enableAEC) |
| try startGraph() |
| try startAudioUnit() |
| } catch { |
| continuation.finish(throwing: error) |
| } |
| } |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| public func startAudioStream(enableAEC: Bool, |
| enableRendererCallback: Bool = false, |
| rendererClosure: ((UnsafeMutablePointer<AudioBufferList>, UInt32) -> Void)? = nil) throws { |
| self.enableRendererCallback = enableRendererCallback |
| try createAUGraphForAudioUnit() |
| try configureAudioUnit() |
| try toggleAudioCancellation(enable: enableAEC) |
| try startGraph() |
| try startAudioUnit() |
| self.rendererClosure = rendererClosure |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| public func stopAudioUnit() throws { |
| var status = AUGraphStop(graph!) |
| guard status == noErr else { |
| logger.error("AUGraphStop failed") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| status = AudioUnitUninitialize(audioUnit!) |
| guard status == noErr else { |
| logger.error("AudioUnitUninitialize failed") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| status = DisposeAUGraph(graph!) |
| guard status == noErr else { |
| logger.error("DisposeAUGraph failed") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| |
| if isVoiceActivityDetectionEnabled { |
| var vadStateAddress = AudioObjectPropertyAddress( |
| mSelector: kAudioDevicePropertyVoiceActivityDetectionState, |
| mScope: kAudioDevicePropertyScopeInput, |
| mElement: kAudioObjectPropertyElementMain |
| ) |
| |
| AudioObjectRemovePropertyListener( |
| deviceID, |
| &vadStateAddress, |
| vadStateListenerCallback, |
| Unmanaged.passUnretained(self).toOpaque() |
| ) |
| } |
| } |
|
|
| private func toggleAudioCancellation(enable: Bool) throws { |
| guard let audioUnit = audioUnit else {return} |
| self.enableAutomaticEchoCancellation = enable |
| |
| var bypassVoiceProcessing: UInt32 = self.enableAutomaticEchoCancellation ? 0 : 1 |
| var status = AudioUnitSetProperty(audioUnit, kAUVoiceIOProperty_BypassVoiceProcessing, kAudioUnitScope_Global, 0, &bypassVoiceProcessing, UInt32(MemoryLayout.size(ofValue: bypassVoiceProcessing))) |
| guard status == noErr else { |
| logger.error("Error in [AudioUnitSetProperty|kAUVoiceIOProperty_BypassVoiceProcessing|kAudioUnitScope_Global]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| var agcVoiceProcessing: UInt32 = self.enableAutomaticEchoCancellation ? 0 : 1 |
| status = AudioUnitSetProperty(audioUnit, kAUVoiceIOProperty_VoiceProcessingEnableAGC, kAudioUnitScope_Global, 0, &agcVoiceProcessing,UInt32(MemoryLayout.size(ofValue: agcVoiceProcessing))) |
| guard status == noErr else { |
| logger.error("Error in [AudioUnitSetProperty|kAUVoiceIOProperty_VoiceProcessingEnableAGC|kAudioUnitScope_Global]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| public func toggleVoiceActivityDetection(enable: Bool) throws { |
| |
| var propertySize = UInt32(MemoryLayout<AudioObjectID>.size) |
| var defaultInputDevice: AudioObjectID = 0 |
| |
| var propertyAddress = AudioObjectPropertyAddress( |
| mSelector: kAudioHardwarePropertyDefaultInputDevice, |
| mScope: kAudioObjectPropertyScopeGlobal, |
| mElement: kAudioObjectPropertyElementMain |
| ) |
| |
| var status = AudioObjectGetPropertyData( |
| AudioObjectID(kAudioObjectSystemObject), |
| &propertyAddress, |
| 0, |
| nil, |
| &propertySize, |
| &defaultInputDevice |
| ) |
| |
| guard status == kAudioHardwareNoError else { |
| logger.error("获取默认输入设备失败") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| self.deviceID = defaultInputDevice |
| |
| |
| var vadEnableAddress = AudioObjectPropertyAddress( |
| mSelector: kAudioDevicePropertyVoiceActivityDetectionEnable, |
| mScope: kAudioDevicePropertyScopeInput, |
| mElement: kAudioObjectPropertyElementMain |
| ) |
| |
| var shouldEnable: UInt32 = enable ? 1 : 0 |
| status = AudioObjectSetPropertyData( |
| deviceID, |
| &vadEnableAddress, |
| 0, |
| nil, |
| UInt32(MemoryLayout<UInt32>.size), |
| &shouldEnable |
| ) |
| |
| guard status == kAudioHardwareNoError else { |
| logger.error("设置VAD状态失败") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| isVoiceActivityDetectionEnabled = enable |
| |
| |
| if enable { |
| var vadStateAddress = AudioObjectPropertyAddress( |
| mSelector: kAudioDevicePropertyVoiceActivityDetectionState, |
| mScope: kAudioDevicePropertyScopeInput, |
| mElement: kAudioObjectPropertyElementMain |
| ) |
| |
| status = AudioObjectAddPropertyListener( |
| deviceID, |
| &vadStateAddress, |
| vadStateListenerCallback, |
| Unmanaged.passUnretained(self).toOpaque() |
| ) |
| |
| guard status == kAudioHardwareNoError else { |
| logger.error("添加VAD状态监听器失败") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| } else { |
| |
| var vadStateAddress = AudioObjectPropertyAddress( |
| mSelector: kAudioDevicePropertyVoiceActivityDetectionState, |
| mScope: kAudioDevicePropertyScopeInput, |
| mElement: kAudioObjectPropertyElementMain |
| ) |
| |
| AudioObjectRemovePropertyListener( |
| deviceID, |
| &vadStateAddress, |
| vadStateListenerCallback, |
| Unmanaged.passUnretained(self).toOpaque() |
| ) |
| } |
| } |
|
|
| private func startGraph() throws { |
| var status = AUGraphInitialize(graph!) |
| guard status == noErr else { |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| status = AUGraphStart(graph!) |
| guard status == noErr else { |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| } |
| |
| private func startAudioUnit() throws { |
| guard let audioUnit = audioUnit else {return} |
| let status = AudioOutputUnitStart(audioUnit) |
| guard AudioOutputUnitStart(audioUnit) == noErr else { |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| } |
| |
| private func createAUGraphForAudioUnit() throws { |
| |
| var status = NewAUGraph(&graph) |
| guard status == noErr else { |
| logger.error("Error in [NewAUGraph]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| |
| var inputcd = AudioComponentDescription() |
| inputcd.componentType = kAudioUnitType_Output |
| inputcd.componentSubType = kAudioUnitSubType_VoiceProcessingIO |
| inputcd.componentManufacturer = kAudioUnitManufacturer_Apple |
| |
| |
| var remoteIONode: AUNode = 0 |
| status = AUGraphAddNode(graph!, &inputcd, &remoteIONode) |
| guard status == noErr else { |
| logger.error("AUGraphAddNode failed") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| |
| status = AUGraphOpen(graph!) |
| guard status == noErr else { |
| logger.error("AUGraphOpen failed") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| |
| status = AUGraphNodeInfo(graph!, remoteIONode, &inputcd, &audioUnit) |
| guard status == noErr else { |
| logger.error("AUGraphNodeInfo failed") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| } |
| |
| |
| |
| |
| static func canonicalStreamDescription(sampleRate: Float64) -> AudioStreamBasicDescription { |
| var canonicalBasicStreamDescription = AudioStreamBasicDescription() |
| canonicalBasicStreamDescription.mSampleRate = sampleRate |
| canonicalBasicStreamDescription.mFormatID = kAudioFormatLinearPCM |
| canonicalBasicStreamDescription.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked |
| canonicalBasicStreamDescription.mFramesPerPacket = 1 |
| canonicalBasicStreamDescription.mChannelsPerFrame = 1 |
| canonicalBasicStreamDescription.mBitsPerChannel = 16 |
| canonicalBasicStreamDescription.mBytesPerPacket = 2 |
| canonicalBasicStreamDescription.mBytesPerFrame = 2 |
| return canonicalBasicStreamDescription |
| } |
| |
| |
| private func configureAudioUnit() throws { |
| guard let audioUnit = audioUnit else {return} |
| |
| let bus_0_output: AudioUnitElement = 0 |
| let bus_1_input: AudioUnitElement = 1 |
| |
| var enableInput: UInt32 = 1 |
| var status = AudioUnitSetProperty(audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, bus_1_input, &enableInput, UInt32(MemoryLayout.size(ofValue: enableInput))) |
| guard status == noErr else { |
| AudioComponentInstanceDispose(audioUnit) |
| logger.error("Error in [AudioUnitSetProperty|kAudioUnitScope_Input]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| var enableOutput: UInt32 = enableRendererCallback ? 1 : 0 |
| status = AudioUnitSetProperty(audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, bus_0_output, &enableOutput, UInt32(MemoryLayout.size(ofValue: enableOutput))) |
| guard status == noErr else { |
| AudioComponentInstanceDispose(audioUnit) |
| logger.error("Error in [AudioUnitSetProperty|kAudioUnitScope_Output]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| status = AudioUnitSetProperty(audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, bus_1_input, &self.streamBasicDescription, UInt32(MemoryLayout<AudioStreamBasicDescription>.size)) |
| guard status == noErr else { |
| AudioComponentInstanceDispose(audioUnit) |
| logger.error("Error in [AudioUnitSetProperty|kAudioUnitProperty_StreamFormat|kAudioUnitScope_Output]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| |
| status = AudioUnitSetProperty(audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, bus_0_output, &self.streamBasicDescription, UInt32(MemoryLayout<AudioStreamBasicDescription>.size)) |
| guard status == noErr else { |
| AudioComponentInstanceDispose(audioUnit) |
| logger.error("Error in [AudioUnitSetProperty|kAudioUnitProperty_StreamFormat|kAudioUnitScope_Input]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| |
| var inputCallbackStruct = AURenderCallbackStruct() |
| inputCallbackStruct.inputProc = kInputCallback |
| inputCallbackStruct.inputProcRefCon = Unmanaged.passUnretained(self).toOpaque() |
| status = AudioUnitSetProperty(audioUnit, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Input, bus_1_input, &inputCallbackStruct, UInt32(MemoryLayout.size(ofValue: inputCallbackStruct))) |
| guard status == noErr else { |
| logger.error("Error in [AudioUnitSetProperty|kAudioOutputUnitProperty_SetInputCallback|kAudioUnitScope_Input]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| |
| if enableRendererCallback { |
| |
| var outputCallbackStruct = AURenderCallbackStruct() |
| outputCallbackStruct.inputProc = kRenderCallback |
| outputCallbackStruct.inputProcRefCon = Unmanaged.passUnretained(self).toOpaque() |
| status = AudioUnitSetProperty(audioUnit, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Output, bus_0_output, &outputCallbackStruct, UInt32(MemoryLayout.size(ofValue: outputCallbackStruct))) |
| guard status == noErr else { |
| logger.error("Error in [AudioUnitSetProperty|kAudioOutputUnitProperty_SetInputCallback|kAudioUnitScope_Output]") |
| throw AECAudioStreamError.osStatusError(status: status) |
| } |
| } |
| } |
| } |
|
|
| |
| private func vadStateListenerCallback( |
| inObjectID: AudioObjectID, |
| inNumberAddresses: UInt32, |
| inAddresses: UnsafePointer<AudioObjectPropertyAddress>, |
| inClientData: UnsafeMutableRawPointer?) -> OSStatus { |
| |
| let audioStream = Unmanaged<AECAudioStream>.fromOpaque(inClientData!).takeUnretainedValue() |
| |
| var vadStateAddress = AudioObjectPropertyAddress( |
| mSelector: kAudioDevicePropertyVoiceActivityDetectionState, |
| mScope: kAudioDevicePropertyScopeInput, |
| mElement: kAudioObjectPropertyElementMain |
| ) |
| |
| var voiceDetected: UInt32 = 0 |
| var propertySize = UInt32(MemoryLayout<UInt32>.size) |
| let status = AudioObjectGetPropertyData( |
| inObjectID, |
| &vadStateAddress, |
| 0, |
| nil, |
| &propertySize, |
| &voiceDetected |
| ) |
| |
| if status == kAudioHardwareNoError { |
| let isVoiceActive = voiceDetected == 1 |
| audioStream.updateVoiceDetectionState(isVoiceActive) |
| } |
| |
| return status |
| } |
|
|
|
|
| private func kInputCallback(inRefCon:UnsafeMutableRawPointer, |
| ioActionFlags:UnsafeMutablePointer<AudioUnitRenderActionFlags>, |
| inTimeStamp:UnsafePointer<AudioTimeStamp>, |
| inBusNumber:UInt32, |
| inNumberFrames:UInt32, |
| ioData:UnsafeMutablePointer<AudioBufferList>?) -> OSStatus { |
| |
| let audioMgr = unsafeBitCast(inRefCon, to: AECAudioStream.self) |
| |
| guard let audioUnit = audioMgr.audioUnit else { |
| return kAudio_ParamError |
| } |
| |
| let audioBuffer = AudioBuffer(mNumberChannels: 1, mDataByteSize: 0, mData: nil) |
| |
| var bufferList = AudioBufferList(mNumberBuffers: 1, mBuffers: audioBuffer) |
| |
| let status = AudioUnitRender(audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList) |
| |
| guard status == noErr else { return status } |
| |
| if let buffer = AVAudioPCMBuffer(pcmFormat: audioMgr.streamFormat, bufferListNoCopy: &bufferList), let captureAudioFrameHandler = audioMgr.capturedFrameHandler { |
| captureAudioFrameHandler(buffer) |
| } |
| return kAudio_ParamError |
| } |
|
|
| private func kRenderCallback(inRefCon:UnsafeMutableRawPointer, |
| ioActionFlags:UnsafeMutablePointer<AudioUnitRenderActionFlags>, |
| inTimeStamp:UnsafePointer<AudioTimeStamp>, |
| inBusNumber:UInt32, |
| inNumberFrames:UInt32, |
| ioData:UnsafeMutablePointer<AudioBufferList>?) -> OSStatus { |
| |
| let audioMgr = unsafeBitCast(inRefCon, to: AECAudioStream.self) |
| |
| guard let outSample = ioData?.pointee.mBuffers.mData?.assumingMemoryBound(to: Int16.self) else { |
| return kAudio_ParamError |
| } |
| let bufferLength = ioData!.pointee.mBuffers.mDataByteSize / UInt32(MemoryLayout<Int16>.stride) |
| |
| memset(outSample, 0, Int(bufferLength)) |
| |
| if let rendererClosure = audioMgr.rendererClosure { |
| rendererClosure(ioData!, inNumberFrames) |
| } else { |
| |
| return kAudioUnitErr_InvalidParameter |
| } |
| |
| return noErr |
| } |
|
|
| private var sharedInstance: AECAudioStream? = nil |
| private var audioDataQueue: AudioDataQueue? = nil |
|
|
| |
| func pcmBufferToData(_ buffer: AVAudioPCMBuffer) -> Data? { |
| let audioBuffer = buffer.audioBufferList.pointee.mBuffers |
| |
| if let mData = audioBuffer.mData { |
| let length = Int(audioBuffer.mDataByteSize) |
| return Data(bytes: mData, count: length) |
| } |
| |
| return nil |
| } |
|
|
| @_cdecl("startRecord") |
| public func startAudioRecord() { |
| if (sharedInstance == nil){ |
| sharedInstance = AECAudioStream(sampleRate: 16000) |
| sharedInstance?.voiceActivityHandler = { isVoiceDetected in |
| if isVoiceDetected { |
| print("检测到语音活动") |
| } else { |
| print("未检测到语音活动") |
| } |
| } |
| } |
| |
| if (audioDataQueue == nil) { |
| audioDataQueue = AudioDataQueue(capacity: 1024) |
| } |
| |
| guard let instance = sharedInstance else { return } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| do { |
| try instance.toggleVoiceActivityDetection(enable: true) |
| } catch { |
|
|
| print("启动VAD失败: \(error)") |
| } |
| |
| Task { |
| for try await pcmBuffer in instance.startAudioStream(enableAEC: true) { |
| if let data = pcmBufferToData(pcmBuffer) { |
| let isVoiceActive = instance.isVoiceDetected |
| _ = audioDataQueue?.push(data: data, isVoiceActive: isVoiceActive) |
| } |
| } |
| |
| |
| |
| } |
| } |
|
|
| @_cdecl("stopRecord") |
| public func stopAudioRecord() { |
| if (sharedInstance == nil) { |
| return |
| } |
| do { |
| try sharedInstance?.stopAudioUnit() |
| } catch { |
| print("停止音频单元失败: \(error)") |
| } |
| |
| } |
|
|
| @_cdecl("getAudioData") |
| public func getAudioData(_ sizePtr: UnsafeMutablePointer<Int>, _ isVoiceActivePtr: UnsafeMutablePointer<Bool>) -> UnsafeMutablePointer<UInt8>? { |
| guard let packet = audioDataQueue?.pop() else { |
| sizePtr.pointee = 0 |
| isVoiceActivePtr.pointee = false |
| return nil |
| } |
| |
| let length = packet.audioData.count |
| sizePtr.pointee = length |
| isVoiceActivePtr.pointee = packet.isVoiceActive |
| |
| let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: length) |
| packet.audioData.copyBytes(to: buffer, count: length) |
| |
| return buffer |
| } |
|
|
|
|
| |
| @_cdecl("freeAudioData") |
| public func freeAudioData(_ buffer: UnsafeMutablePointer<UInt8>?) { |
| buffer?.deallocate() |
| } |
|
|