-
-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
503 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| // | ||
| // AudioManager.swift | ||
| // | ||
| // | ||
| // Created by Tatsuya Tanaka on 2022/03/06. | ||
| // | ||
|
|
||
| import AVFAudio | ||
| import VCamLogger | ||
|
|
||
| public final class AudioManager { | ||
| public init() {} | ||
|
|
||
| public var onUpdateAudioBuffer: ((AVAudioPCMBuffer, AVAudioTime, TimeInterval) -> Void) = { _, _, _ in } | ||
|
|
||
| public var isRunning: Bool { | ||
| audioEngine.isRunning | ||
| } | ||
|
|
||
| private var audioEngine = AVAudioEngine() | ||
|
|
||
| public func startRecording(onStart: @escaping (AVAudioFormat) -> Void) throws { | ||
| guard DeviceAuthorization.authorizationStatus(for: .mic) else { | ||
| Logger.log("requestAuthorization") | ||
| DeviceAuthorization.requestAuthorization(type: .mic) { [self] authorized in | ||
| guard authorized else { return } | ||
| DispatchQueue.main.async { [self] in | ||
| try? startRecording(onStart: onStart) | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| Task { @MainActor in | ||
| // After changing settings with CoreAudio, a delay is needed to prevent installTap failures | ||
| try? await Task.sleep(for: .milliseconds(500)) | ||
|
|
||
| audioEngine = AVAudioEngine() | ||
| guard audioEngine.inputNode.inputFormat(forBus: 0).sampleRate != 0 else { | ||
| return | ||
| } | ||
|
|
||
| let inputNode = audioEngine.inputNode | ||
| let recordingFormat = inputNode.inputFormat(forBus: 0) | ||
|
|
||
| Logger.log("installTap") | ||
| inputNode.installTap(onBus: 0, | ||
| bufferSize: 1024, | ||
| format: recordingFormat) { [weak self] (buffer: AVAudioPCMBuffer, when: AVAudioTime) in | ||
| guard let self = self else { return } | ||
| // https://stackoverflow.com/questions/26115626/i-want-to-call-20-times-per-second-the-installtaponbusbuffersizeformatblock | ||
| // Matching the bufferSize prevents audio from intermittently cutting out during recording. | ||
| buffer.frameLength = 1024 | ||
| self.onUpdateAudioBuffer(buffer, when, inputNode.presentationLatency) | ||
| } | ||
|
|
||
| try? audioEngine.start() | ||
| onStart(recordingFormat) | ||
| } | ||
| } | ||
|
|
||
| public func stopRecording() { | ||
| Logger.log("") | ||
| audioEngine.stop() | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| // | ||
| // AvatarAudioManager.swift | ||
| // | ||
| // | ||
| // Created by Tatsuya Tanaka on 2022/03/06. | ||
| // | ||
|
|
||
| import Foundation | ||
| import VCamData | ||
| import VCamMedia | ||
| import VCamEntity | ||
| import VCamBridge | ||
| import VCamLogger | ||
| import AVFAudio | ||
|
|
||
| public final class AvatarAudioManager: NSObject { | ||
| public static let shared = AvatarAudioManager() | ||
|
|
||
| public var videoRecorderRenderAudioFrame: (AVAudioPCMBuffer, AVAudioTime, TimeInterval, AudioDevice?) -> Void = { _, _, _, _ in } | ||
|
|
||
| private let audioManager = AudioManager() | ||
| private let audioExpressionEstimator = AudioExpressionEstimator() | ||
| private var usage = Usage() | ||
| private var isConfiguring = false | ||
|
|
||
| public var currentInputDevice: AudioDevice? { | ||
| guard let uid = UserDefaults.standard.value(for: .audioDeviceUid) else { return .defaultDevice() } | ||
| return AudioDevice.device(forUid: uid) | ||
| } | ||
|
|
||
| override init() { | ||
| super.init() | ||
|
|
||
| NotificationCenter.default.addObserver( | ||
| self, selector: #selector(onConfigurationChange), name: .AVAudioEngineConfigurationChange, | ||
| object: nil) | ||
|
|
||
| if let device = currentInputDevice { | ||
| setAudioDevice(device) | ||
| } | ||
| } | ||
|
|
||
| public func startIfNeeded() { | ||
| guard !UniBridge.shared.lipSyncWebCam.wrappedValue else { return } | ||
| AvatarAudioManager.shared.start(usage: .lipSync) | ||
| } | ||
|
|
||
| public func start(usage: Usage, isSystemSoundRecording: Bool = false) { | ||
| reconfigureIfNeeded() | ||
| do { | ||
| Logger.log("\(isConfiguring), \(audioManager.isRunning)") | ||
| if !isConfiguring, !audioManager.isRunning { | ||
| // There's a delay in AudioManager::startRecording, so don't allow consecutive calls (it causes a crash in installTap) | ||
| isConfiguring = true | ||
|
|
||
| if isSystemSoundRecording { | ||
| AudioDevice.device(forUid: "vcam-audio-device-001")?.setAsDefaultDevice() | ||
| } else { | ||
| currentInputDevice?.setAsDefaultDevice() | ||
| } | ||
| try audioManager.startRecording { [self] inputFormat in | ||
| audioExpressionEstimator.configure(format: inputFormat) | ||
| isConfiguring = false | ||
| } | ||
| } | ||
| self.usage.insert(usage) | ||
| } catch { | ||
| isConfiguring = false | ||
| Logger.error(error) | ||
| } | ||
| } | ||
|
|
||
| public func stop(usage: Usage) { | ||
| self.usage.remove(usage) | ||
| guard self.usage.isEmpty else { return } | ||
| audioManager.stopRecording() | ||
| audioExpressionEstimator.reset() | ||
| } | ||
|
|
||
| private func reconfigureIfNeeded() { | ||
| setEmotionEnabled(UserDefaults.standard.value(for: .useEmotion)) | ||
| audioExpressionEstimator.onAudioLevelUpdate = { level in | ||
| Task { @MainActor in | ||
| UniBridge.shared.micAudioLevel.wrappedValue = CGFloat(level) | ||
| } | ||
| } | ||
| audioManager.onUpdateAudioBuffer = { buffer, time, latency in | ||
| if Self.shared.usage.contains(.lipSync) { // Ensure no malfunctions during recording | ||
| Self.shared.audioExpressionEstimator.analyze(buffer: buffer, time: time) | ||
| } | ||
| Self.shared.videoRecorderRenderAudioFrame(buffer, time, latency, Self.shared.currentInputDevice) | ||
| } | ||
| } | ||
|
|
||
| public func setEmotionEnabled(_ isEnabled: Bool) { | ||
| audioExpressionEstimator.onUpdate = isEnabled ? { emotion in | ||
| Task { @MainActor in | ||
| UniBridge.shared.facialExpression(emotion.rawValue) | ||
| } | ||
| } : nil | ||
| } | ||
|
|
||
| public func setAudioDevice(_ audioDevice: AudioDevice) { | ||
| Logger.log(audioDevice.name()) | ||
| UserDefaults.standard.set(audioDevice.uid, for: .audioDeviceUid) | ||
|
|
||
| if audioManager.isRunning { | ||
| let usage = self.usage | ||
| stop(usage: usage) | ||
| Task { @MainActor in | ||
| try? await Task.sleep(for: .milliseconds(500)) | ||
| start(usage: usage) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @objc private func onConfigurationChange(notification: Notification) { | ||
| // guard audioManager.isRunning else { return } | ||
| // stop() | ||
| // DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { [self] in | ||
| // start() | ||
| // } | ||
| } | ||
|
|
||
| public struct Usage: OptionSet { | ||
| public init(rawValue: UInt) { | ||
| self.rawValue = rawValue | ||
| } | ||
|
|
||
| public var rawValue: UInt = 0 | ||
|
|
||
| public static let lipSync = Usage(rawValue: 0x1) | ||
| public static let record = Usage(rawValue: 0x2) | ||
| public static let all: Usage = [lipSync, record] | ||
| } | ||
| } |
Oops, something went wrong.