From c6ed47b6493eff3320a215747a77348c6a92b448 Mon Sep 17 00:00:00 2001 From: "tattn (Tatsuya Tanaka)" Date: Mon, 9 Oct 2023 03:31:38 +0900 Subject: [PATCH] Open source --- app/xcode/Package.swift | 2 +- .../Sources/VCamBridge/RenderTexture.swift | 2 +- .../Sources/VCamMedia/AudioDevice+.swift | 164 ++++++++++++++++++ app/xcode/Sources/VCamUI/AudioManager.swift | 66 +++++++ .../Sources/VCamUI/AvatarAudioManager.swift | 136 +++++++++++++++ app/xcode/Sources/VCamUI/Migration.swift | 117 +++++++++++++ .../Sources/VCamUI/RenderTextureManager.swift | 9 + .../Sources/VCamUI/SceneObjectManager.swift | 9 + 8 files changed, 503 insertions(+), 2 deletions(-) create mode 100644 app/xcode/Sources/VCamUI/AudioManager.swift create mode 100644 app/xcode/Sources/VCamUI/AvatarAudioManager.swift create mode 100644 app/xcode/Sources/VCamUI/Migration.swift diff --git a/app/xcode/Package.swift b/app/xcode/Package.swift index b03c880..7962331 100644 --- a/app/xcode/Package.swift +++ b/app/xcode/Package.swift @@ -24,7 +24,7 @@ let package = Package( .define("ENABLE_MOCOPI") ]), .target(name: "VCamLocalization", resources: [.process("VCamResources")]), - .target(name: "VCamMedia", dependencies: ["VCamEntity", "VCamAppExtension"]), + .target(name: "VCamMedia", dependencies: ["VCamEntity", "VCamAppExtension", "VCamLogger"]), .target(name: "VCamBridge", dependencies: ["VCamUIFoundation"]), .target(name: "VCamCamera", dependencies: ["VCamData", "VCamLogger"]), .target(name: "VCamTracking", dependencies: ["VCamData", "VCamLogger"]), diff --git a/app/xcode/Sources/VCamBridge/RenderTexture.swift b/app/xcode/Sources/VCamBridge/RenderTexture.swift index e6016d8..8e4d8d8 100644 --- a/app/xcode/Sources/VCamBridge/RenderTexture.swift +++ b/app/xcode/Sources/VCamBridge/RenderTexture.swift @@ -35,7 +35,7 @@ public final class MainTexture { } } -func __bridge(_ ptr: UnsafeRawPointer) -> T { +public func __bridge(_ ptr: UnsafeRawPointer) -> T { Unmanaged.fromOpaque(ptr).takeUnretainedValue() } diff --git a/app/xcode/Sources/VCamMedia/AudioDevice+.swift b/app/xcode/Sources/VCamMedia/AudioDevice+.swift index 4c57885..271657f 100644 --- a/app/xcode/Sources/VCamMedia/AudioDevice+.swift +++ b/app/xcode/Sources/VCamMedia/AudioDevice+.swift @@ -7,7 +7,9 @@ import Foundation import CoreAudio +import AVFAudio import VCamEntity +import VCamLogger extension AudioDevice { public init(id: AudioDeviceID) { @@ -149,3 +151,165 @@ public extension AudioDevice { return buf } } + +extension AudioDevice { + private static var cachedDevices: [AudioDevice] = [] { + didSet { + NotificationCenter.default.post(name: .deviceWasChanged, object: nil) + } + } + + public static func configure() { + Self.cachedDevices = Self.loadDevices() + + NotificationCenter.default.addObserver(forName: .AVCaptureDeviceWasConnected, object: nil, queue: .main) { _ in + Self.cachedDevices = Self.loadDevices() + } + + NotificationCenter.default.addObserver(forName: .AVCaptureDeviceWasDisconnected, object: nil, queue: .main) { _ in + Self.cachedDevices = Self.loadDevices() + } + } + + public static func devices() -> [AudioDevice] { + cachedDevices + } + + private static func loadDevices() -> [AudioDevice] { + var propsize: UInt32 = 0 + + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioDevicePropertyScopeInput, + mElement: AudioObjectPropertyElement(kAudioObjectPropertyElementMain)) + + var result = AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), + &address, + UInt32(MemoryLayout.size), + nil, &propsize) + + if result != 0 { + Logger.log("Error \(result) from AudioObjectGetPropertyDataSize") + return [] + } + + let deviceCount = Int(propsize / UInt32(MemoryLayout.size)) + + if deviceCount == 0 { + return [] + } + + var devids = [AudioDeviceID](repeating: 0, count: deviceCount) + + result = 0 + devids.withUnsafeMutableBufferPointer { bufferPointer in + if let pointer = bufferPointer.baseAddress { + result = AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), + &address, + 0, + nil, + &propsize, + pointer) + } + } + + if result != 0 { + Logger.log("Error \(result) from AudioObjectGetPropertyData") + return [] + } + + return devids.map { + AudioDevice(id: $0) + } + .filter { + let name = $0.name() + return !name.hasPrefix("CADefaultDevice") && !name.hasPrefix("vcam-audio-device") + } + .filter { $0.isMicrophone() } + } + + public static func device(forUid uid: String) -> AudioDevice? { + devices().first { $0.uid == uid } + } + + public func setAsDefaultDevice() { + Logger.log(name()) + + var outputID: AudioDeviceID = id + let propsize = UInt32(MemoryLayout.size) + let selector = isMicrophone() ? kAudioHardwarePropertyDefaultInputDevice : kAudioHardwarePropertyDefaultOutputDevice + var address = AudioObjectPropertyAddress(mSelector: selector, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMain) + let error = AudioObjectSetPropertyData(AudioObjectID(kAudioObjectSystemObject), + &address, + 0, + nil, + propsize, + &outputID) + if error != noErr { + Logger.log("defaultDevice error: \(error)") + } + } + + private func isMicrophone() -> Bool { + // https://stackoverflow.com/questions/4575408/audioobjectgetpropertydata-to-get-a-list-of-input-devices + var streamConfigAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioDevicePropertyScopeInput, + mElement: 0) + + var propertySize = UInt32(0) + + var result = AudioObjectGetPropertyDataSize(id, &streamConfigAddress, 0, nil, &propertySize) + if result != 0 { + Logger.log("Error \(result) from AudioObjectGetPropertyDataSize") + return false + } + + let audioBufferList = AudioBufferList.allocate(maximumBuffers: Int(propertySize)) + defer { + free(audioBufferList.unsafeMutablePointer) + } + result = AudioObjectGetPropertyData(id, &streamConfigAddress, 0, nil, &propertySize, audioBufferList.unsafeMutablePointer) + if result != 0 { + Logger.log("Error \(result) from AudioObjectGetPropertyDataSize") + return false + } + + var channelCount = 0 + for i in 0 ..< Int(audioBufferList.unsafeMutablePointer.pointee.mNumberBuffers) { + channelCount = channelCount + Int(audioBufferList[i].mNumberChannels) + } + + return channelCount > 0 + } +} + +extension AudioUnit { + public func getDeviceId() -> AudioDeviceID { + var outputID: AudioDeviceID = 0 + var propsize = UInt32(MemoryLayout.size) + let error = AudioUnitGetProperty(self, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, + 0, + &outputID, + &propsize) + if error != noErr { + Logger.log("getDeviceID error: \(error)") + } + return outputID + } + + public func set(_ device: AudioDevice) { + // https://www.hackingwithswift.com/forums/macos/how-do-you-specify-the-audio-output-device-on-a-mac-in-swift/13177 + var inputDeviceID = device.id + let status = AudioUnitSetProperty(self, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, + 0, + &inputDeviceID, + UInt32(MemoryLayout.size)) + print("AudioUnit.set", status) + } +} + diff --git a/app/xcode/Sources/VCamUI/AudioManager.swift b/app/xcode/Sources/VCamUI/AudioManager.swift new file mode 100644 index 0000000..f81f417 --- /dev/null +++ b/app/xcode/Sources/VCamUI/AudioManager.swift @@ -0,0 +1,66 @@ +// +// AudioManager.swift +// +// +// Created by Tatsuya Tanaka on 2022/03/06. +// + +import AVFAudio +import VCamLogger + +public final class AudioManager { + public init() {} + + public var onUpdateAudioBuffer: ((AVAudioPCMBuffer, AVAudioTime, TimeInterval) -> Void) = { _, _, _ in } + + public var isRunning: Bool { + audioEngine.isRunning + } + + private var audioEngine = AVAudioEngine() + + public func startRecording(onStart: @escaping (AVAudioFormat) -> Void) throws { + guard DeviceAuthorization.authorizationStatus(for: .mic) else { + Logger.log("requestAuthorization") + DeviceAuthorization.requestAuthorization(type: .mic) { [self] authorized in + guard authorized else { return } + DispatchQueue.main.async { [self] in + try? startRecording(onStart: onStart) + } + } + return + } + + Task { @MainActor in + // After changing settings with CoreAudio, a delay is needed to prevent installTap failures + try? await Task.sleep(for: .milliseconds(500)) + + audioEngine = AVAudioEngine() + guard audioEngine.inputNode.inputFormat(forBus: 0).sampleRate != 0 else { + return + } + + let inputNode = audioEngine.inputNode + let recordingFormat = inputNode.inputFormat(forBus: 0) + + Logger.log("installTap") + inputNode.installTap(onBus: 0, + bufferSize: 1024, + format: recordingFormat) { [weak self] (buffer: AVAudioPCMBuffer, when: AVAudioTime) in + guard let self = self else { return } + // https://stackoverflow.com/questions/26115626/i-want-to-call-20-times-per-second-the-installtaponbusbuffersizeformatblock + // Matching the bufferSize prevents audio from intermittently cutting out during recording. + buffer.frameLength = 1024 + self.onUpdateAudioBuffer(buffer, when, inputNode.presentationLatency) + } + + try? audioEngine.start() + onStart(recordingFormat) + } + } + + public func stopRecording() { + Logger.log("") + audioEngine.stop() + } +} diff --git a/app/xcode/Sources/VCamUI/AvatarAudioManager.swift b/app/xcode/Sources/VCamUI/AvatarAudioManager.swift new file mode 100644 index 0000000..6b7bfd0 --- /dev/null +++ b/app/xcode/Sources/VCamUI/AvatarAudioManager.swift @@ -0,0 +1,136 @@ +// +// AvatarAudioManager.swift +// +// +// Created by Tatsuya Tanaka on 2022/03/06. +// + +import Foundation +import VCamData +import VCamMedia +import VCamEntity +import VCamBridge +import VCamLogger +import AVFAudio + +public final class AvatarAudioManager: NSObject { + public static let shared = AvatarAudioManager() + + public var videoRecorderRenderAudioFrame: (AVAudioPCMBuffer, AVAudioTime, TimeInterval, AudioDevice?) -> Void = { _, _, _, _ in } + + private let audioManager = AudioManager() + private let audioExpressionEstimator = AudioExpressionEstimator() + private var usage = Usage() + private var isConfiguring = false + + public var currentInputDevice: AudioDevice? { + guard let uid = UserDefaults.standard.value(for: .audioDeviceUid) else { return .defaultDevice() } + return AudioDevice.device(forUid: uid) + } + + override init() { + super.init() + + NotificationCenter.default.addObserver( + self, selector: #selector(onConfigurationChange), name: .AVAudioEngineConfigurationChange, + object: nil) + + if let device = currentInputDevice { + setAudioDevice(device) + } + } + + public func startIfNeeded() { + guard !UniBridge.shared.lipSyncWebCam.wrappedValue else { return } + AvatarAudioManager.shared.start(usage: .lipSync) + } + + public func start(usage: Usage, isSystemSoundRecording: Bool = false) { + reconfigureIfNeeded() + do { + Logger.log("\(isConfiguring), \(audioManager.isRunning)") + if !isConfiguring, !audioManager.isRunning { + // There's a delay in AudioManager::startRecording, so don't allow consecutive calls (it causes a crash in installTap) + isConfiguring = true + + if isSystemSoundRecording { + AudioDevice.device(forUid: "vcam-audio-device-001")?.setAsDefaultDevice() + } else { + currentInputDevice?.setAsDefaultDevice() + } + try audioManager.startRecording { [self] inputFormat in + audioExpressionEstimator.configure(format: inputFormat) + isConfiguring = false + } + } + self.usage.insert(usage) + } catch { + isConfiguring = false + Logger.error(error) + } + } + + public func stop(usage: Usage) { + self.usage.remove(usage) + guard self.usage.isEmpty else { return } + audioManager.stopRecording() + audioExpressionEstimator.reset() + } + + private func reconfigureIfNeeded() { + setEmotionEnabled(UserDefaults.standard.value(for: .useEmotion)) + audioExpressionEstimator.onAudioLevelUpdate = { level in + Task { @MainActor in + UniBridge.shared.micAudioLevel.wrappedValue = CGFloat(level) + } + } + audioManager.onUpdateAudioBuffer = { buffer, time, latency in + if Self.shared.usage.contains(.lipSync) { // Ensure no malfunctions during recording + Self.shared.audioExpressionEstimator.analyze(buffer: buffer, time: time) + } + Self.shared.videoRecorderRenderAudioFrame(buffer, time, latency, Self.shared.currentInputDevice) + } + } + + public func setEmotionEnabled(_ isEnabled: Bool) { + audioExpressionEstimator.onUpdate = isEnabled ? { emotion in + Task { @MainActor in + UniBridge.shared.facialExpression(emotion.rawValue) + } + } : nil + } + + public func setAudioDevice(_ audioDevice: AudioDevice) { + Logger.log(audioDevice.name()) + UserDefaults.standard.set(audioDevice.uid, for: .audioDeviceUid) + + if audioManager.isRunning { + let usage = self.usage + stop(usage: usage) + Task { @MainActor in + try? await Task.sleep(for: .milliseconds(500)) + start(usage: usage) + } + } + } + + @objc private func onConfigurationChange(notification: Notification) { +// guard audioManager.isRunning else { return } +// stop() +// DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { [self] in +// start() +// } + } + + public struct Usage: OptionSet { + public init(rawValue: UInt) { + self.rawValue = rawValue + } + + public var rawValue: UInt = 0 + + public static let lipSync = Usage(rawValue: 0x1) + public static let record = Usage(rawValue: 0x2) + public static let all: Usage = [lipSync, record] + } +} diff --git a/app/xcode/Sources/VCamUI/Migration.swift b/app/xcode/Sources/VCamUI/Migration.swift new file mode 100644 index 0000000..a86947f --- /dev/null +++ b/app/xcode/Sources/VCamUI/Migration.swift @@ -0,0 +1,117 @@ +// +// Migration.swift +// +// +// Created by Tatsuya Tanaka on 2022/03/13. +// + +import Foundation +import VCamData +import VCamEntity +import VCamLogger +import VCamCamera +import AppKit + +public struct Migration { + public static var openVirtualCameraPreference: () -> Void = {} + + public static func migrate() async { + let previousVersion = UserDefaults.standard.value(for: .previousVersion) + + do { + try await migrationFirst(previousVersion: previousVersion) + try migration095(previousVersion: previousVersion) + try await migration0110(previousVersion: previousVersion) + } catch { + Logger.error(error) + } + + UserDefaults.standard.set(Bundle.main.version, for: .previousVersion) + } +} + +extension Migration { + static func migrationFirst(previousVersion: String) async throws { + guard previousVersion.isEmpty else { return } + await VCamAlert.showModal(title: L10n.installVirtualCamera.text, message: L10n.explainAboutInstallingCameraExtension.text, canCancel: false) + Task { + do { + if CoreMediaSinkStream.isInstalled { + NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy")!) + } + try await CameraExtension().installExtension() + _ = await VirtualCameraManager.shared.installAndStartCameraExtension() + await VCamAlert.showModal(title: L10n.success.text, message: L10n.restartAfterInstalling.text, canCancel: false) + } catch { + await VCamAlert.showModal(title: L10n.failure.text, message: L10n.failedToInstallCameraExtension.text, canCancel: false) + } + } + } + + static func migration095(previousVersion: String) throws { + guard previousVersion == "0.9.4" else { return } // only for 0.9.4 + Logger.log("") + + var metadata = try VCamShortcutMetadata.load() + guard metadata.version == 1 else { return } + + let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory()) + for id in metadata.ids { + let url = URL.shortcutDirectory(id: id) + let temporaryURL = temporaryDirectoryURL.appendingPathComponent(id.uuidString) + do { + try FileManager.default.moveItem(at: url, to: temporaryURL) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + try FileManager.default.moveItem(at: temporaryURL, to: .shortcutData(id: id)) + + try FileManager.default.createDirectory(at: .shortcutResourceDirectory(id: id), withIntermediateDirectories: true) + } catch { + Logger.error(error) + } + } + + metadata.version = 2 + try metadata.save() + } + + @MainActor + static func migration0110(previousVersion: String) async throws { + guard LegacyPluginHelper.isPluginInstalled() else { + return + } + + let version = previousVersion.components(separatedBy: ".").compactMap(Int.init) + guard version.count == 3, version[0] == 0, version[1] < 12 else { + return + } + Logger.log("") + + let isNewCameraInstalled = CoreMediaSinkStream.isInstalled + + await VCamAlert.showModal( + title: isNewCameraInstalled ? L10n.deleteOldDALPlugin.text : L10n.migrateToNewVirtualCamera.text, + message: isNewCameraInstalled ? L10n.deleteOldDALPluginMessage.text : L10n.migrateToNewVirtualCameraMessage.text, + canCancel: false + ) + + while LegacyPluginHelper.isPluginInstalled() { + await LegacyPluginHelper.uninstallPlugin(canCancel: false) + } + + if !isNewCameraInstalled { + openVirtualCameraPreference() + } + } + + private static func installCameraExtension() async { + do { + try await CameraExtension().installExtensionIfNotInstalled() + } catch { + await VCamAlert.showModal( + title: L10n.failure.text, + message: L10n.failedToInstallCameraPlugin.text, + canCancel: false + ) + } + } +} diff --git a/app/xcode/Sources/VCamUI/RenderTextureManager.swift b/app/xcode/Sources/VCamUI/RenderTextureManager.swift index 6e70d61..3323c4d 100644 --- a/app/xcode/Sources/VCamUI/RenderTextureManager.swift +++ b/app/xcode/Sources/VCamUI/RenderTextureManager.swift @@ -9,6 +9,15 @@ import SwiftUI import VCamBridge import VCamLogger +@_cdecl("uniOnAddTexture") +public func uniOnAddTexture(_ id: Int32, imagePointer: UnsafeRawPointer?) { + guard let pointer = imagePointer else { return } + let bridgedMtlTexture: any MTLTexture = __bridge(pointer) + let mtlTexture = bridgedMtlTexture.makeTextureView(pixelFormat: .rgba8Unorm)! // Do not use sRGB as it becomes brighter due to gamma correction + + RenderTextureManager.shared.setRenderTexture(mtlTexture, id: id) +} + public final class RenderTextureManager { public static let shared = RenderTextureManager() diff --git a/app/xcode/Sources/VCamUI/SceneObjectManager.swift b/app/xcode/Sources/VCamUI/SceneObjectManager.swift index 88ed6dd..a544c30 100644 --- a/app/xcode/Sources/VCamUI/SceneObjectManager.swift +++ b/app/xcode/Sources/VCamUI/SceneObjectManager.swift @@ -281,3 +281,12 @@ extension SceneObjectManager { } } } + +extension SceneObjectManager { + public func addImage(url: URL) { + let renderer = ImageRenderer(imageURL: url, filter: nil) + let id = RenderTextureManager.shared.add(renderer) + let canvasSize = UniBridge.shared.canvasCGSize + add(.init(id: id, type: .image(.init(url: url, size: .init(width: renderer.size.width / canvasSize.width, height: renderer.size.height / canvasSize.height), filter: nil)), isHidden: false, isLocked: false)) + } +}