From ad27e6d6a91e25850085a0f320891250a7ca3a17 Mon Sep 17 00:00:00 2001 From: "tattn (Tatsuya Tanaka)" Date: Mon, 9 Oct 2023 02:13:18 +0900 Subject: [PATCH] Open source --- app/xcode/Sources/VCamBridge/UniBridge.swift | 75 +-- .../VCamUI/Extensions/UniBridge+.swift | 64 +++ .../Sources/VCamUI/Extensions/UniSet.swift | 33 ++ .../Sources/VCamUI/RenderTextureManager.swift | 97 ++++ app/xcode/Sources/VCamUI/SceneManager.swift | 213 ++++++++ .../Sources/VCamUI/SceneObjectManager.swift | 283 +++++++++++ app/xcode/Sources/VCamUI/ScreenRecorder.swift | 468 ++++++++++++++++++ .../Action/VCamBlendShapeAction.swift | 2 +- .../Shortcut/Action/VCamLoadSceneAction.swift | 4 +- .../Shortcut/Action/VCamMessageAction.swift | 2 +- .../Shortcut/Action/VCamMotionAction.swift | 26 +- .../Shortcut/VCamShortcutBuilderView.swift | 8 +- app/xcode/Sources/VCamUI/UniAction.swift | 40 +- app/xcode/Sources/VCamUI/UniState.swift | 72 +-- app/xcode/Sources/VCamUI/WebRenderer.swift | 356 +++++++++++++ 15 files changed, 1625 insertions(+), 118 deletions(-) create mode 100644 app/xcode/Sources/VCamUI/Extensions/UniBridge+.swift create mode 100644 app/xcode/Sources/VCamUI/Extensions/UniSet.swift create mode 100644 app/xcode/Sources/VCamUI/RenderTextureManager.swift create mode 100644 app/xcode/Sources/VCamUI/SceneManager.swift create mode 100644 app/xcode/Sources/VCamUI/SceneObjectManager.swift create mode 100644 app/xcode/Sources/VCamUI/ScreenRecorder.swift create mode 100644 app/xcode/Sources/VCamUI/WebRenderer.swift diff --git a/app/xcode/Sources/VCamBridge/UniBridge.swift b/app/xcode/Sources/VCamBridge/UniBridge.swift index d135f08..1837e79 100644 --- a/app/xcode/Sources/VCamBridge/UniBridge.swift +++ b/app/xcode/Sources/VCamBridge/UniBridge.swift @@ -1,7 +1,8 @@ /// GENERATED BY ./scripts/generate_bridge import SwiftUI public final class UniBridge { - public init() {} + public static let shared = UniBridge() + private init() {} public enum IntType: Int32 { case lensFlare = 0 case facialExpression = 1 @@ -10,7 +11,7 @@ public final class UniBridge { } public let intMapper = ValueBinding() public var lensFlare: Binding { intMapper.binding(.lensFlare) } - public lazy var facialExpression = intMapper.set(.facialExpression) + public private(set) lazy var facialExpression = intMapper.set(.facialExpression) public var objectSelected: Binding { intMapper.binding(.objectSelected) } public var qualityLevel: Binding { intMapper.binding(.qualityLevel) } @@ -88,9 +89,9 @@ public final class UniBridge { public var useCombineMesh: Binding { boolMapper.binding(.useCombineMesh) } public var useAddToMacOSMenuBar: Binding { boolMapper.binding(.useAddToMacOSMenuBar) } public var useVSync: Binding { boolMapper.binding(.useVSync) } - public lazy var useHandTracking = boolMapper.set(.useHandTracking) - public lazy var useBlinker = boolMapper.set(.useBlinker) - public lazy var useFullTracking = boolMapper.set(.useFullTracking) + public private(set) lazy var useHandTracking = boolMapper.set(.useHandTracking) + public private(set) lazy var useBlinker = boolMapper.set(.useBlinker) + public private(set) lazy var useFullTracking = boolMapper.set(.useFullTracking) public var lipSyncWebCam: Binding { boolMapper.binding(.lipSyncWebCam) } public var interactable: Bool { boolMapper.get(.interactable) } public var hasPerfectSyncBlendShape: Bool { boolMapper.get(.hasPerfectSyncBlendShape) } @@ -112,11 +113,11 @@ public final class UniBridge { } public let stringMapper = ValueBinding() public var message: Binding { stringMapper.binding(.message) } - public lazy var loadVRM = stringMapper.set(.loadVRM) - public lazy var loadModel = stringMapper.set(.loadModel) + public private(set) lazy var loadVRM = stringMapper.set(.loadVRM) + public private(set) lazy var loadModel = stringMapper.set(.loadModel) public var currentDisplayParameter: Binding { stringMapper.binding(.currentDisplayParameter) } public var allDisplayParameterPresets: String { stringMapper.get(.allDisplayParameterPresets) } - public lazy var showEmojiStamp = stringMapper.set(.showEmojiStamp) + public private(set) lazy var showEmojiStamp = stringMapper.set(.showEmojiStamp) public var blendShapes: String { stringMapper.get(.blendShapes) } public var currentBlendShape: Binding { stringMapper.binding(.currentBlendShape) } @@ -140,23 +141,23 @@ public final class UniBridge { case quitApp = 16 } public let triggerMapper = ValueBinding() - public lazy var openVRoidHub = triggerMapper.trigger(.openVRoidHub) - public lazy var resetCamera = triggerMapper.trigger(.resetCamera) - public lazy var motionJump = triggerMapper.trigger(.motionJump) - public lazy var motionWhat = triggerMapper.trigger(.motionWhat) - public lazy var motionHello = triggerMapper.trigger(.motionHello) - public lazy var motionYear = triggerMapper.trigger(.motionYear) - public lazy var motionWin = triggerMapper.trigger(.motionWin) - public lazy var applyDisplayParameter = triggerMapper.trigger(.applyDisplayParameter) - public lazy var saveDisplayParameter = triggerMapper.trigger(.saveDisplayParameter) - public lazy var addDisplayParameter = triggerMapper.trigger(.addDisplayParameter) - public lazy var deleteDisplayParameter = triggerMapper.trigger(.deleteDisplayParameter) - public lazy var deleteObject = triggerMapper.trigger(.deleteObject) - public lazy var resetAllObjects = triggerMapper.trigger(.resetAllObjects) - public lazy var editAvatar = triggerMapper.trigger(.editAvatar) - public lazy var pauseApp = triggerMapper.trigger(.pauseApp) - public lazy var resumeApp = triggerMapper.trigger(.resumeApp) - public lazy var quitApp = triggerMapper.trigger(.quitApp) + public private(set) lazy var openVRoidHub = triggerMapper.trigger(.openVRoidHub) + public private(set) lazy var resetCamera = triggerMapper.trigger(.resetCamera) + public private(set) lazy var motionJump = triggerMapper.trigger(.motionJump) + public private(set) lazy var motionWhat = triggerMapper.trigger(.motionWhat) + public private(set) lazy var motionHello = triggerMapper.trigger(.motionHello) + public private(set) lazy var motionYear = triggerMapper.trigger(.motionYear) + public private(set) lazy var motionWin = triggerMapper.trigger(.motionWin) + public private(set) lazy var applyDisplayParameter = triggerMapper.trigger(.applyDisplayParameter) + public private(set) lazy var saveDisplayParameter = triggerMapper.trigger(.saveDisplayParameter) + public private(set) lazy var addDisplayParameter = triggerMapper.trigger(.addDisplayParameter) + public private(set) lazy var deleteDisplayParameter = triggerMapper.trigger(.deleteDisplayParameter) + public private(set) lazy var deleteObject = triggerMapper.trigger(.deleteObject) + public private(set) lazy var resetAllObjects = triggerMapper.trigger(.resetAllObjects) + public private(set) lazy var editAvatar = triggerMapper.trigger(.editAvatar) + public private(set) lazy var pauseApp = triggerMapper.trigger(.pauseApp) + public private(set) lazy var resumeApp = triggerMapper.trigger(.resumeApp) + public private(set) lazy var quitApp = triggerMapper.trigger(.quitApp) public enum StructType: Int32 { case backgroundColor = 0 @@ -207,18 +208,18 @@ public final class UniBridge { } } public let arrayMapper = ValueBinding() - public lazy var headTransform = arrayMapper.set(.headTransform, type: [Float].self) - public lazy var hands = arrayMapper.set(.hands, type: [Float].self) - public lazy var fingers = arrayMapper.set(.fingers, type: [Float].self) - public lazy var receiveVCamBlendShape = arrayMapper.set(.receiveVCamBlendShape, type: [Float].self) - public lazy var receivePerfectSync = arrayMapper.set(.receivePerfectSync, type: [Float].self) - public lazy var addRenderTexture = arrayMapper.set(.addRenderTexture, type: [Int32].self) - public lazy var updateRenderTexture = arrayMapper.set(.updateRenderTexture, type: [Int32].self) - public lazy var updateObjectOrder = arrayMapper.set(.updateObjectOrder, type: [Int32].self) - public lazy var setObjectActive = arrayMapper.set(.setObjectActive, type: [Int32].self) - public lazy var setObjectLocked = arrayMapper.set(.setObjectLocked, type: [Int32].self) - public lazy var objectAvatarTransform = arrayMapper.set(.objectAvatarTransform, type: [Float].self) - public lazy var addWind = arrayMapper.set(.addWind, type: [Int32].self) + public private(set) lazy var headTransform = arrayMapper.set(.headTransform, type: [Float].self) + public private(set) lazy var hands = arrayMapper.set(.hands, type: [Float].self) + public private(set) lazy var fingers = arrayMapper.set(.fingers, type: [Float].self) + public private(set) lazy var receiveVCamBlendShape = arrayMapper.set(.receiveVCamBlendShape, type: [Float].self) + public private(set) lazy var receivePerfectSync = arrayMapper.set(.receivePerfectSync, type: [Float].self) + public private(set) lazy var addRenderTexture = arrayMapper.set(.addRenderTexture, type: [Int32].self) + public private(set) lazy var updateRenderTexture = arrayMapper.set(.updateRenderTexture, type: [Int32].self) + public private(set) lazy var updateObjectOrder = arrayMapper.set(.updateObjectOrder, type: [Int32].self) + public private(set) lazy var setObjectActive = arrayMapper.set(.setObjectActive, type: [Int32].self) + public private(set) lazy var setObjectLocked = arrayMapper.set(.setObjectLocked, type: [Int32].self) + public private(set) lazy var objectAvatarTransform = arrayMapper.set(.objectAvatarTransform, type: [Float].self) + public private(set) lazy var addWind = arrayMapper.set(.addWind, type: [Int32].self) public var canvasSize: [Float] { arrayMapper.get(.canvasSize, size: 2) } public var screenResolution: Binding<[Int32]> { arrayMapper.binding(.screenResolution, size: 2) } diff --git a/app/xcode/Sources/VCamUI/Extensions/UniBridge+.swift b/app/xcode/Sources/VCamUI/Extensions/UniBridge+.swift new file mode 100644 index 0000000..61b61e9 --- /dev/null +++ b/app/xcode/Sources/VCamUI/Extensions/UniBridge+.swift @@ -0,0 +1,64 @@ +// +// UniBridge+.swift +// +// +// Created by Tatsuya Tanaka on 2022/05/23. +// + +import Foundation +import VCamEntity +import SwiftUI +import VCamBridge + +extension UniState { + public init(_ state: CustomState) { + get = state.get + set = state.set + name = state.name + reloadThrottle = state.reloadThrottle + } + + public struct CustomState { + public init(get: @escaping () -> Value, set: @escaping (Value) -> Void, name: String = "", reloadThrottle: Bool = false) { + self.get = get + self.set = set + self.name = name + self.reloadThrottle = reloadThrottle + } + + public var get: () -> Value + public var set: (Value) -> Void + public var name = "" + public var reloadThrottle = false + } +} + +extension UniState.CustomState { + public static var typedScreenResolution: Self { + let rawValue = UniState<[Int32]>(.screenResolution, name: "screenResolution", as: [Int32].self) + return .init { + let size = rawValue.wrappedValue + guard size.count == 2 else { return .init(width: 1920, height: 1280) } // an empty array after disposal + return ScreenResolution(width: Int(size[0]), height: Int(size[1])) + } set: { + let isLandscape = MainTexture.shared.isLandscape + rawValue.wrappedValue = [Int32($0.size.width), Int32($0.size.height)] + if isLandscape != MainTexture.shared.isLandscape { + SceneManager.shared.changeAspectRatio() + } + } + } +} + +extension UniBridge { + public var canvasCGSize: CGSize { + let size = canvasSize + guard size.count == 2 else { return .init(width: 1920, height: 1280) } // an empty array after disposal + return .init(width: CGFloat(size[0]), height: CGFloat(size[1])) + } + + public static var cachedBlendShapes: [String] = [] + public var cachedBlendShapes: [String] { + Self.cachedBlendShapes + } +} diff --git a/app/xcode/Sources/VCamUI/Extensions/UniSet.swift b/app/xcode/Sources/VCamUI/Extensions/UniSet.swift new file mode 100644 index 0000000..26c7eec --- /dev/null +++ b/app/xcode/Sources/VCamUI/Extensions/UniSet.swift @@ -0,0 +1,33 @@ +// +// UniSet.swift +// +// +// Created by Tatsuya Tanaka on 2023/03/01. +// + +import SwiftUI +import VCamEntity +import VCamBridge + +@propertyWrapper public struct UniSet: DynamicProperty { + public init(_ type: UniBridge.BoolType, name: String) where Value == Bool { + let mapper = UniBridge.shared.boolMapper + self.set = mapper.set(type) + self.name = name + } + + private let set: (Value) -> Void + private var name = "" + + public var wrappedValue: Action { + .init(set: set) + } + + public struct Action { + let set: (T) -> Void + + public func callAsFunction(_ value: T) { + set(value) + } + } +} diff --git a/app/xcode/Sources/VCamUI/RenderTextureManager.swift b/app/xcode/Sources/VCamUI/RenderTextureManager.swift new file mode 100644 index 0000000..6e70d61 --- /dev/null +++ b/app/xcode/Sources/VCamUI/RenderTextureManager.swift @@ -0,0 +1,97 @@ +// +// RenderTextureManager.swift +// +// +// Created by Tatsuya Tanaka on 2022/03/20. +// + +import SwiftUI +import VCamBridge +import VCamLogger + +public final class RenderTextureManager { + public static let shared = RenderTextureManager() + + private var recorders: [Int32: any RenderTextureRenderer] = [:] + private let ciContext = CIContext(mtlDevice: MTLCreateSystemDefaultDevice()!) + + public func add(_ recorder: any RenderTextureRenderer) -> Int32 { + let id = Int32.random(in: 0.. (any RenderTextureRenderer)? { + recorders[id] + } + + public func setRenderTexture(_ texture: any MTLTexture, id: Int32) { + uniDebugLog("setRenderTexture: \(id) in \(recorders.keys)") + guard let recorder = recorders[id] else { + uniDebugLog("setRenderTexture: no recorder \(id)") + return + } + Logger.log("\(texture.width)x\(texture.height), \(type(of: recorder))") + + recorder.setRenderTexture { [self] image in + let width = image.extent.width + let height = image.extent.height + if recorder.updateTextureSizeIfNeeded(imageWidth: width, imageHeight: height) { + Logger.log("updateTextureSizeIfNeeded") + // iPhone's screen size initially becomes 0x0, so reconfigure when a texture is retrieved. + if let object = SceneObjectManager.shared.objects.find(byId: id), let texture = object.type.croppableTexture { + texture.crop = recorder.cropRect + texture.region = .init(origin: .zero, size: .invalid) + recorder.disableRenderTexture() + Task { @MainActor in + SceneObjectManager.shared.update(object) + } + return + } + } + + let (camWidth, camHeight) = (Int(width * recorder.cropRect.width), Int(width * recorder.cropRect.height)) + if texture.width == camWidth, texture.height == camHeight { + let croppedImage = recorder.cropped(of: image) + ciContext.render(croppedImage, to: texture, commandBuffer: nil, bounds: croppedImage.extent, colorSpace: .sRGB) + } else { + Logger.log("setRenderTexture change size: \(texture.width) == \(camWidth), \(texture.height) == \(camHeight), \(width)") + // Update the texture size + recorder.disableRenderTexture() + Task { @MainActor in + UniBridge.shared.updateRenderTexture([Int32(id), Int32(camWidth), Int32(camHeight)]) + } + } + } + } + + func remove(id: Int32) { + guard let recorder = recorders[id] else { return } + recorder.stopRendering() + recorders.removeValue(forKey: id) + } + + func removeAll() { + let ids = [Int32](recorders.keys) + for id in ids { + remove(id: id) + } + } + + public func pause() { + for recorder in recorders.values { + recorder.pauseRendering() + } + } + + public func resume() { + for recorder in recorders.values { + recorder.resumeRendering() + } + } +} diff --git a/app/xcode/Sources/VCamUI/SceneManager.swift b/app/xcode/Sources/VCamUI/SceneManager.swift new file mode 100644 index 0000000..11f3ddd --- /dev/null +++ b/app/xcode/Sources/VCamUI/SceneManager.swift @@ -0,0 +1,213 @@ +// +// SceneManager.swift +// +// +// Created by Tatsuya Tanaka on 2022/05/07. +// + +import SwiftUI +import VCamEntity +import VCamBridge +import VCamData +import VCamLogger + +@_cdecl("uniLoadScene") +public func uniLoadScene() { + do { + try SceneManager.shared.loadCurrentScene() + } catch { + uniDebugLog(error.localizedDescription) + } +} + +@_cdecl("uniUpdateScene") +public func uniUpdateScene() { + do { + try SceneManager.shared.saveCurrentSceneAndObjects() + } catch { + uniDebugLog(error.localizedDescription) + } +} + +public final class SceneManager: ObservableObject { + public static let shared = SceneManager() + + @Published public private(set) var currentSceneId: Int32 + + var currentScene: VCamScene { + get throws { + try scenes.find(byId: currentSceneId).orThrow(NSError.vcam(message: "invalid scene id: \(currentSceneId)")) + } + } + + @Published public var scenes: [VCamScene] + private var landScapeScenes: [VCamScene] + private var portraitScenes: [VCamScene] + + private init() { + let newSceneId = Int32.random(in: 0.. VCamScene { + let dataStore = VCamSceneDataStore(sceneId: sceneId) + return dataStore.makeNewScene() + } + + public func addNewScene() throws { + let scene = Self.createNewScene() + try add(scene) + try loadScene(id: scene.id) + } + + public func add(_ scene: VCamScene) throws { + Logger.log("") + let dataStore = VCamSceneDataStore(sceneId: scene.id) + try dataStore.save(scene) + scenes.append(scene) + } + + public func update(_ scene: VCamScene) { + guard let index = scenes.index(ofId: scene.id) else { + return + } + scenes[index] = scene + } + + public func remove(byId id: Int32) { + guard let scene = scenes.find(byId: id) else { + return + } + remove(scene) + } + + private func remove(_ scene: VCamScene) { + scenes.remove(byId: scene.id) + try? VCamSceneDataStore(sceneId: scene.id).delete() + try? save() + if currentSceneId == scene.id, let nextScene = scenes.first { + try? loadScene(id: nextScene.id) + } else { + uniUpdateScene() + } + } + + public func move(byId id: Int32, up: Bool) { + guard let index = scenes.index(ofId: id) else { + return + } + + let destination = index + (up ? 1 : -1) + if 0 <= destination && destination < scenes.count { + scenes.swapAt(index, destination) + try? save() + } + } + + public func move(fromOffsets source: IndexSet, toOffset destination: Int) { + scenes.move(fromOffsets: source, toOffset: destination) + try? save() + } + + public func loadScene(id: Int32) throws { + if currentSceneId != id { + try? saveCurrentSceneAndObjects() // Save when switching scenes + } + let scene = try scenes.find(byId: id).orThrow(NSError.vcam(message: "invalid scene id: \(id)")) + uniDebugLog("\(scene.id) \(scene.objects.count)") + Logger.log("\(scene.id) \(scene.objects.count)") + currentSceneId = id + SceneObjectManager.shared.loadObjects(scene) + } + + public func loadCurrentScene() throws { + try loadScene(id: currentSceneId) + } + + func saveCurrentSceneAndObjects() throws { + let dataStore = VCamSceneDataStore(sceneId: currentSceneId) + let scene = try dataStore.makeScene(name: currentScene.name, objects: SceneObjectManager.shared.objects) + try dataStore.save(scene) + update(scene) + } + + private func save() throws { + var metadata = VCamSceneMetadata.loadOrCreate() + if MainTexture.shared.isLandscape { + landScapeScenes = scenes + } else { + portraitScenes = scenes + } + metadata.sceneIds = (landScapeScenes + portraitScenes).map(\.id) + try metadata.save() + } + + func changeAspectRatio() { + // Store the previous state in memory + // (since the flag is already toggled, store it in the reversed state) + if MainTexture.shared.isLandscape { + portraitScenes = scenes + } else { + landScapeScenes = scenes + } + scenes = MainTexture.shared.isLandscape ? landScapeScenes : portraitScenes + + if let scene = scenes.first { + UniBridge.shared.resetAllObjects() // Since processing is delayed, first remove only the list items from UI. + Task { @MainActor in + try? await Task.sleep(for: .milliseconds(800)) + // A slight delay is needed until the canvas size aspect ratio changes (TODO: investigation required). + try? self.loadScene(id: scene.id) + } + } + } +} + +private extension Array where Element == VCamScene { + func scenes(isLandscape: Bool) -> Self { + return filter { + guard let aspectRatio = $0.aspectRatio else { + return isLandscape + } + if isLandscape { + return aspectRatio <= 1 + } else { + return aspectRatio > 1 + } + } + } +} diff --git a/app/xcode/Sources/VCamUI/SceneObjectManager.swift b/app/xcode/Sources/VCamUI/SceneObjectManager.swift new file mode 100644 index 0000000..88ed6dd --- /dev/null +++ b/app/xcode/Sources/VCamUI/SceneObjectManager.swift @@ -0,0 +1,283 @@ +// +// SceneObjectManager.swift +// +// +// Created by Tatsuya Tanaka on 2022/04/27. +// + +import SwiftUI +import VCamEntity +import VCamLocalization +import VCamBridge +import VCamData +import VCamLogger +import AVFoundation + +@_cdecl("uniRemoveObject") +public func uniRemoveObject(_ id: Int32) { + uniDebugLog("uniRemoveTexture \(id)") + SceneObjectManager.shared.remove(byId: id) +} + +@_cdecl("uniUpdateObjectAvatar") +public func uniUpdateObjectAvatar(px: Float, py: Float, pz: Float, rx: Float, ry: Float, rz: Float) { + guard let object = SceneObjectManager.shared.objects.find(byId: SceneObject.avatarID), + case .avatar(let avatar) = object.type else { return } + avatar.position = .init(px, py, pz) + avatar.rotation = .init(rx, ry, rz) + uniUpdateScene() +} + +@_cdecl("uniUpdateObjectImage") +public func uniUpdateObjectImage(id: Int32, px: Float, py: Float, sx: Float, sy: Float) { + guard let object = SceneObjectManager.shared.objects.find(byId: id) else { return } + switch object.type { + case .avatar, .wind: + return + case let .image(image): + image.offset = .init(px, py) + image.size = CGSize(width: CGFloat(sx), height: CGFloat(sy)) + case let .screen(screen): + screen.region.origin = .init(x: CGFloat(px), y: CGFloat(py)) + screen.region.size = .init(width: CGFloat(sx), height: CGFloat(sy)) + case let .videoCapture(videoCapture): + videoCapture.region.origin = .init(x: CGFloat(px), y: CGFloat(py)) + videoCapture.region.size = .init(width: CGFloat(sx), height: CGFloat(sy)) + case let .web(web): + web.region.origin = .init(x: CGFloat(px), y: CGFloat(py)) + web.region.size = .init(width: CGFloat(sx), height: CGFloat(sy)) + } + uniUpdateScene() +} + +public final class SceneObjectManager: ObservableObject { + public static let shared = SceneObjectManager() + + @Published public var objects: [SceneObject] = VCamSceneDataStore.defaultObjects + + public func add(_ object: SceneObject) { + Logger.log("") + configure(object) + objects.append(object) + uniUpdateScene() + } + + private func configure(_ object: SceneObject) { + let uniBridge = UniBridge.shared + uniDebugLog("SceneObjectManager.configure: \(object)") + switch object.type { + case .avatar: () + case let .image(image): + let sceneId = SceneManager.shared.currentSceneId + image.url = VCamSceneDataStore(sceneId: sceneId).copyData(fromURL: image.url) + + let canvasSize = uniBridge.canvasCGSize + + if image.size == .zero { // Migration from v0.6.3 and below & change the aspect ratio + image.size = NSImage(contentsOf: image.url)?.size ?? .init(width: 800, height: 800) + image.size = .init(width: image.size.width / canvasSize.width, height: image.size.height / canvasSize.height) + } + + let region: CGRect + if image.offset.x < -1000 { // Set the initial position to be dependent on textureRect + region = .init(origin: .zero, size: .invalid) + } else { + region = .init(origin: .init(x: CGFloat(image.offset.x), y: CGFloat(image.offset.y)), size: image.size) + } + + let rect = textureRect(region: region, crop: .init(x: 0, y: 0, width: 1, height: canvasSize.height * image.size.height / (canvasSize.width * image.size.width))) + image.offset = .init(x: Float(rect[0]) / Float(canvasSize.width), y: Float(rect[1]) / Float(canvasSize.height)) + image.size = .init(width: CGFloat(rect[2]) / canvasSize.width, height: CGFloat(rect[3]) / canvasSize.height) + uniBridge.addRenderTexture([object.id, RenderTextureType.photo.rawValue, rect[2], rect[3]] + rect) + case let .screen(screen): + let rect = textureRect(region: screen.region, crop: screen.crop) + uniBridge.addRenderTexture([object.id, RenderTextureType.screen.rawValue, rect[2], rect[3]] + rect) + case let .videoCapture(videoCapture): + let rect = textureRect(region: videoCapture.region, crop: videoCapture.crop) + uniBridge.addRenderTexture([object.id, RenderTextureType.captureDevice.rawValue, rect[2], rect[3]] + rect) + case let .web(web): + let rect = textureRect(region: web.region, crop: web.crop) + uniBridge.addRenderTexture([object.id, RenderTextureType.web.rawValue, rect[2], rect[3]] + rect) + case let .wind(wind): + let direction = wind.direction + let scale: Float = 100000 // Shift the digits by the number of significant figures to send as Int. + uniBridge.addWind([object.id, Int32(direction.x * scale), Int32(direction.y * scale), Int32(direction.z * scale)]) + } + + uniBridge.setObjectActive([object.id, object.isHidden ? 0 : 1]) + uniBridge.setObjectLocked([object.id, object.isLocked ? 1 : 0]) + } + + public func update(_ object: SceneObject) { + objects.update(object) + configure(object) + uniUpdateScene() + } + + func remove(byIndex index: Int) { + remove(objects[index]) + } + + public func remove(byId id: Int32) { + guard let object = objects.find(byId: id) else { + return + } + remove(object) + } + + private func remove(_ object: SceneObject) { + Logger.log("\(object.type)") + switch object.type { + case .avatar: + return + case .wind: () + case let .image(image): + try? FileManager.default.removeItem(at: image.url) + RenderTextureManager.shared.remove(id: object.id) + case .screen, .videoCapture, .web: + RenderTextureManager.shared.remove(id: object.id) + } + objects.remove(byId: object.id) + UniBridge.shared.deleteObject() + uniUpdateScene() + } + + public func move(byId id: Int32, up: Bool) { + guard let index = objects.index(ofId: id) else { + return + } + Logger.log("\(id), \(up), \(index)") + + let destination = index + (up ? 1 : -1) + if 0 <= destination && destination < objects.count { + objects.swapAt(index, destination) + updateObjectOrder() + } + } + + public func moveToBack(id: Int32) { + guard let index = objects.index(ofId: id) else { + return + } + Logger.log("\(index)") + + let element = objects.remove(at: index) + objects.insert(element, at: 0) + updateObjectOrder() + } + + public func move(fromOffsets source: IndexSet, toOffset destination: Int) { + Logger.log("\(source), \(destination)") + objects.move(fromOffsets: source, toOffset: destination) + updateObjectOrder() + } + + func updateObjectOrder() { + UniBridge.shared.updateObjectOrder(SceneObjectManager.shared.objects.map(\.id) + [-1]) + uniUpdateScene() + } + + public func dispose() { + objects = objects.filter { $0.id == SceneObject.avatarID } + RenderTextureManager.shared.removeAll() + } + + private func textureRect(region: CGRect, crop: CGRect) -> [Int32] { + let canvasSize = UniBridge.shared.canvasCGSize + uniDebugLog("textureRect: r\(region), c\(crop), s\(canvasSize)") + let x = Int32(canvasSize.width * region.origin.x) + let y = Int32(canvasSize.height * region.origin.y) + + let estimatedWidth = fitSize(crop.size, regionSize: region.size).width + let estimatedHeight = estimatedWidth * crop.size.height / crop.size.width + + return [x, y, Int32(estimatedWidth), Int32(estimatedHeight)] + } + + private func fitSize(_ size: CGSize, regionSize: CGSize) -> CGSize { + let canvasSize = UniBridge.shared.canvasCGSize + + var estimatedWidth: CGFloat + + if regionSize.width < 0 { // Can't compare with .invalid, so determine based on whether it's less than 0 + // Initially, display at 80% relative to the canvas to fit within the screen. + if size.width > size.height { + estimatedWidth = canvasSize.width * 0.8 + } else { + let height = canvasSize.height * 0.8 + estimatedWidth = height * size.width / size.height + } + } else { + estimatedWidth = canvasSize.width * regionSize.width + } + return .init(width: estimatedWidth, height: estimatedWidth * size.height / size.width) + } +} + +extension SceneObjectManager { + func loadObjects(_ scene: VCamScene) { + let dataStore = VCamSceneDataStore(sceneId: scene.id) + + RenderTextureManager.shared.removeAll() + UniBridge.shared.resetAllObjects() + + self.objects = [] + + let group = DispatchGroup() + scene.objects.forEach { _ in group.enter() } + + for object in scene.objects { + let sceneObject = object.sceneObject(dataStore: dataStore) + switch object.type { + case let .avatar(avatar): + Logger.log("load avatar \(avatar == .zero)") + if avatar == .zero { + UniBridge.shared.resetCamera() + } else { + UniBridge.shared.objectAvatarTransform([ + avatar.position.x, avatar.position.y, avatar.position.z, + avatar.rotation.x, avatar.rotation.y, avatar.rotation.z, + ]) + } + group.leave() + case .image: + if case let .image(image) = sceneObject.type { + let renderer = ImageRenderer(imageURL: image.url, filter: image.filter) + RenderTextureManager.shared.set(renderer, id: object.id) + configure(sceneObject) + } + group.leave() + case let .screen(id, state): + ScreenRecorder.create(id: id, screenCapture: state) { recorder in + recorder.filter = state.texture.filter.map(ImageFilter.init(configuration:)) + RenderTextureManager.shared.set(recorder, id: object.id) + self.configure(sceneObject) + group.leave() + } + case let .captureDevice(uniqueID, state): + if let device = AVCaptureDevice(uniqueID: uniqueID), + let drawer = try? CaptureDeviceRenderer(device: device, cropRect: state.crop.rect) { + drawer.filter = state.filter.map(ImageFilter.init(configuration:)) + RenderTextureManager.shared.set(drawer, id: object.id) + } + configure(sceneObject) + group.leave() + case let .web(state): + let renderer = WebRenderer(resource: state.url != nil ? .url(state.url!) : .path(bookmark: state.path ?? .init()), size: state.texture.textureSize, fps: state.fps, css: state.css, js: state.js) + renderer.filter = state.texture.filter.map(ImageFilter.init(configuration:)) + RenderTextureManager.shared.set(renderer, id: object.id) + configure(sceneObject) + group.leave() + case .wind: + configure(sceneObject) + group.leave() + } + objects.append(sceneObject) + } + + group.notify(queue: .main) { + Logger.log("finish loadObjects") + self.updateObjectOrder() + } + } +} diff --git a/app/xcode/Sources/VCamUI/ScreenRecorder.swift b/app/xcode/Sources/VCamUI/ScreenRecorder.swift new file mode 100644 index 0000000..516703c --- /dev/null +++ b/app/xcode/Sources/VCamUI/ScreenRecorder.swift @@ -0,0 +1,468 @@ +// +// ScreenRecorder.swift +// +// +// Created by Tatsuya Tanaka on 2022/03/20. +// + +import ScreenCaptureKit +import AVFAudio +import VCamBridge +import VCamEntity + +public protocol ScreenRecorderProtocol: AnyObject { + @MainActor func stopCapture() async +} + +public final class ScreenRecorder: NSObject, ObservableObject, ScreenRecorderProtocol { + public enum CaptureType { + case independentWindow + case display + + init(type: VCamScene.ScreenCapture.CaptureType) { + switch type { + case .window: self = .independentWindow + case .display: self = .display + } + } + } + + public struct CaptureConfiguration { + public init( + captureType: ScreenRecorder.CaptureType = .display, + display: SCDisplay? = nil, + window: SCWindow? = nil, + filterOutOwningApplication: Bool = true, + capturesAudio: Bool = true, + minimumFrameInterval: CMTime? = nil + ) { + self.captureType = captureType + self.display = display + self.window = window + self.filterOutOwningApplication = filterOutOwningApplication + self.capturesAudio = capturesAudio + self.minimumFrameInterval = minimumFrameInterval + } + + public var captureType: CaptureType = .display + public var display: SCDisplay? + public var window: SCWindow? + public var filterOutOwningApplication = true + public var capturesAudio = true + public var minimumFrameInterval: CMTime? + + public var id: String? { + switch captureType { + case .independentWindow: + return window?.id + case .display: + return display?.id + } + } + } + + struct CapturedFrame { + var sampleBuffer: CMSampleBuffer + var surfaceRef: IOSurfaceRef + var contentRect: CGRect + var displayTime: TimeInterval + var contentScale: Double + var scaleFactor: Double + var surface: IOSurface { + // Force-cast the IOSurfaceRef to IOSurface. + return unsafeBitCast(surfaceRef, to: IOSurface.self) + } + + var croppedCIImage: CIImage { + CIImage(ioSurface: surfaceRef).cropped(to: contentRect.applying(.init(scaleX: scaleFactor, y: scaleFactor))) + } + } + + struct ScreenRecorderError: Error { + let errorDescription: String + + init(_ description: String) { + errorDescription = description + } + } + + private var didVideoOutput: ((CapturedFrame) -> Void)? + private var didAudioOutput: ((CMSampleBuffer) -> Void)? + + public var size: CGSize { + guard let config = captureConfig else { + return .init(width: 1024, height: 640) + } + if config.captureType == .display, let display = config.display { + let size = CGDisplayScreenSize(display.displayID) + return .init(width: Int(size.width), height: Int(size.height)) + } else if let window = config.window { + let scale = NSApp.window(withWindowNumber: Int(window.windowID))?.backingScaleFactor ?? 2 + let frame = window.frame + return .init(width: Int(frame.width * scale), height: Int(frame.height * scale)) + } + return .init(width: 1024, height: 640) + } + + public var cropRect = CGRect(x: 0, y: 0, width: 1024, height: 640) + + public var filter: ImageFilter? + + @MainActor @Published private(set) var latestFrame: CapturedFrame? + @MainActor @Published private(set) var error: (any Error)? + @MainActor @Published private(set) var isRecording = false + + public private(set) var captureConfig: CaptureConfiguration? + private var stream: SCStream? + private var cpuStartTime = mach_absolute_time() + private var mediaStartTime = CACurrentMediaTime() + private let videoSampleBufferQueue = DispatchQueue(label: "com.github.tattn.vcam.queue.screenrecorder.video") + private let audioSampleBufferQueue = DispatchQueue(label: "com.github.tattn.vcam.queue.screenrecorder.audio") + + @MainActor + public func startCapture(with captureConfig: CaptureConfiguration) async { + error = nil + isRecording = false + self.captureConfig = captureConfig + + do { + // Create the content filter with the sample app settings. + let filter = try await contentFilter(for: captureConfig) + + // Create the stream configuration with the sample app settings. + let streamConfig = streamConfiguration(for: captureConfig) + + // Create a capture stream with the filter and stream configuration. + stream = SCStream(filter: filter, configuration: streamConfig, delegate: self) + + // Add a stream output to capture screen content. + try stream?.addStreamOutput(self, type: .screen, sampleHandlerQueue: videoSampleBufferQueue) + if captureConfig.capturesAudio { + try stream?.addStreamOutput(self, type: .audio, sampleHandlerQueue: audioSampleBufferQueue) + } + + // Start the capture session. + try await stream?.startCapture() + + cpuStartTime = mach_absolute_time() + mediaStartTime = CACurrentMediaTime() + isRecording = true + + await update(with: captureConfig) + } catch { + uniDebugLog("ScreenCapture error: \(error)") + self.error = error + } + } + + @MainActor + public func update(with captureConfig: CaptureConfiguration) async { + do { + self.captureConfig = captureConfig + let filter = try await contentFilter(for: captureConfig) + let streamConfig = streamConfiguration(for: captureConfig) + try await stream?.updateConfiguration(streamConfig) + try await stream?.updateContentFilter(filter) + } catch { + self.error = error + } + } + + @MainActor + func refreshScreen() async { + try? await stream?.stopCapture() + try? await stream?.startCapture() + } + + @MainActor + public func stopCapture() async { + isRecording = false + + do { + try await stream?.stopCapture() + } catch { + self.error = error + } + } + + private func contentFilter(for config: CaptureConfiguration) async throws -> SCContentFilter { + switch config.captureType { + case .display: + if let display = config.display { + + // Create a content filter that includes all content from the display, + // excluding the sample app's window. + if config.filterOutOwningApplication { + + // Get the content that's available to capture. + let content = try await SCShareableContent.excludingDesktopWindows(false, + onScreenWindowsOnly: true) + + // Exclude the sample app by matching the bundle identifier. + let excludedApps = content.applications.filter { app in + Bundle.main.bundleIdentifier == app.bundleIdentifier + } + + // Create a content filter that excludes the sample app. + return SCContentFilter(display: display, + excludingApplications: excludedApps, + exceptingWindows: []) + + } else { + // Create a content filter that includes the entire display. + return SCContentFilter(display: display, excludingWindows: []) + } + } + case .independentWindow: + if let window = config.window { + + // Create a content filter that includes a single window. + return SCContentFilter(desktopIndependentWindow: window) + + } + } + throw ScreenRecorderError("The configuration doesn't provide a display or window.") + } + + private func streamConfiguration(for captureConfig: CaptureConfiguration) -> SCStreamConfiguration { + let streamConfig = SCStreamConfiguration() + + streamConfig.capturesAudio = captureConfig.capturesAudio + streamConfig.sampleRate = 44100 // not working? + streamConfig.channelCount = 1 +// streamConfig.excludesCurrentProcessAudio = isAppAudioExcluded // if excludes + + if let minimumFrameInterval = captureConfig.minimumFrameInterval { + streamConfig.minimumFrameInterval = minimumFrameInterval + } + + // Set the capture size to twice the display size to support retina displays. + if let display = captureConfig.display, captureConfig.captureType == .display { + streamConfig.width = display.width * 2 + streamConfig.height = display.height * 2 + } + + // Set the capture interval at 60 fps. + streamConfig.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(60)) + + // Increase the depth of the frame queue to ensure high fps at the expense of increasing + // the memory footprint of WindowServer. + streamConfig.queueDepth = 5 + + return streamConfig + } + + private func convertToSeconds(_ machTime: UInt64) -> TimeInterval { + var timebase = mach_timebase_info_data_t() + mach_timebase_info(&timebase) + let nanoseconds = machTime * UInt64(timebase.numer) / UInt64(timebase.denom) + return Double(nanoseconds) / Double(kSecondScale) + } +} + +extension ScreenRecorder: SCStreamOutput { + public func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard sampleBuffer.isValid else { + return + } + + if type == .screen { + guard let frame = createCapturedFrame(for: sampleBuffer) else { + return + } + DispatchQueue.main.async { + self.latestFrame = frame + self.didVideoOutput?(frame) + } + } else if type == .audio { +// guard let buffer = createPCMBuffer(for: sampleBuffer) else { +// return +// } + DispatchQueue.main.async { + self.didAudioOutput?(sampleBuffer) + } + } + } + + private func createCapturedFrame(for sampleBuffer: CMSampleBuffer) -> CapturedFrame? { + guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: true) as? [[SCStreamFrameInfo: Any]], + let attachments = attachmentsArray.first else { + return nil + } + + guard let statusRawValue = attachments[SCStreamFrameInfo.status] as? Int, + let status = SCFrameStatus(rawValue: statusRawValue), + status == .complete else { + return nil + } + + guard let pixelBuffer = sampleBuffer.imageBuffer else { + return nil + } + + guard let surfaceRef = CVPixelBufferGetIOSurface(pixelBuffer)?.takeUnretainedValue() else { + return nil + } + + guard let contentRectDict = attachments[.contentRect], + let contentRect = CGRect(dictionaryRepresentation: contentRectDict as! CFDictionary) else { + return nil + } + + guard let displayTime = attachments[.displayTime] as? UInt64 else { + return nil + } + + let elapsedTime = convertToSeconds(displayTime) - convertToSeconds(cpuStartTime) + + guard let contentScale = attachments[.contentScale] as? Double, + let scaleFactor = attachments[.scaleFactor] as? Double else { + return nil + } + + return CapturedFrame(sampleBuffer: sampleBuffer, + surfaceRef: surfaceRef, + contentRect: contentRect, + displayTime: elapsedTime, + contentScale: contentScale, + scaleFactor: scaleFactor) + } + + private func createPCMBuffer(for sampleBuffer: CMSampleBuffer) -> AVAudioPCMBuffer? { + var ablPointer: UnsafePointer? + try? sampleBuffer.withAudioBufferList(flags: .audioBufferListAssure16ByteAlignment) { audioBufferList, blockBuffer in + ablPointer = audioBufferList.unsafePointer + } + guard let audioBufferList = ablPointer, + let absd = sampleBuffer.formatDescription?.audioStreamBasicDescription, + let format = AVAudioFormat(standardFormatWithSampleRate: absd.mSampleRate, channels: absd.mChannelsPerFrame) else { return nil } + return AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: audioBufferList) + } +} + +extension ScreenRecorder: SCStreamDelegate { + public func stream(_ stream: SCStream, didStopWithError error: any Error) { + DispatchQueue.main.async { + self.error = error + self.isRecording = false + } + } +} + +extension ScreenRecorder: RenderTextureRenderer { + public func setRenderTexture(updator: @escaping (CIImage) -> Void) { + didVideoOutput = { [weak self] frame in + guard let self = self else { return } + DispatchQueue.main.async { + var image = frame.croppedCIImage + image = self.filter?.apply(to: image) ?? image + updator(image) + } + } + Task { + await refreshScreen() // Call this because if not updated, the screen may become transparent when added. + } + } + + @MainActor + public func snapshot() -> CIImage { + guard let frame = latestFrame else { return .init() } + return frame.croppedCIImage + } + + public func disableRenderTexture() { + didVideoOutput = nil + } + + public func pauseRendering() { + Task { + await stopCapture() + } + } + + public func resumeRendering() { + guard let captureConfig = captureConfig else { return } + Task { + await startCapture(with: captureConfig) + } + } + + public func stopRendering() { + didVideoOutput = nil + Task { + await stopCapture() + } + } +} + +public extension ScreenRecorder { + static func create(id: String, screenCapture: VCamScene.ScreenCapture, completion: @escaping (ScreenRecorder) -> Void) { + Task { @MainActor in // Use the main thread for size since the Unity side's Canvas size is required + let availableContent = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + uniDebugLog("ScreenRecorder.create: \(availableContent)") + let configuration = CaptureConfiguration( + captureType: .init(type: screenCapture.captureType), + display: availableContent.displays.first { $0.id == id }, + window: availableContent.windows.first { $0.id == id } + ) + + let screenRecorder = ScreenRecorder() + screenRecorder.cropRect = screenCapture.texture.crop.rect + screenRecorder.filter = screenCapture.texture.filter.map(ImageFilter.init(configuration:)) + await screenRecorder.startCapture(with: configuration) + uniDebugLog("ScreenRecorder.create: \(screenRecorder)") + completion(screenRecorder) + } + } + + static func audioOnly(output: @escaping (CMSampleBuffer, CFTimeInterval) -> Void) -> ScreenRecorder { + let audioCapture = ScreenRecorder() + Task { + let availableContent = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + let configuration = ScreenRecorder.CaptureConfiguration( + captureType: .display, + display: availableContent.displays.first, // If not set to display, sound will not be recorded. + minimumFrameInterval: .init(value: 1, timescale: 10) // https://developer.apple.com/forums/thread/718279 + ) + await audioCapture.startCapture(with: configuration) + } + let mediaStartTime = audioCapture.mediaStartTime + audioCapture.didAudioOutput = { buffer in + output(buffer, mediaStartTime) + } + return audioCapture + } +} + +extension SCDisplay: Identifiable { + public var id: String { + return String(CGDisplaySerialNumber(displayID)) + } +} + +extension SCWindow: Identifiable { + public var id: String { + guard let infoList = CGWindowListCopyWindowInfo(.optionIncludingWindow, windowID) as? [NSDictionary], + let info = infoList.first, + let ownerName = info[kCGWindowOwnerName] as? String, + let title = info[kCGWindowName] as? String else { + return "" + } + return "\(ownerName)-\(title)" + } +} + +public extension ScreenRecorder.CaptureType { + var type: VCamScene.ScreenCapture.CaptureType { + switch self { + case .independentWindow: return .window + case .display: return .display + } + } +} diff --git a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamBlendShapeAction.swift b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamBlendShapeAction.swift index 6a5e315..dc2bce1 100644 --- a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamBlendShapeAction.swift +++ b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamBlendShapeAction.swift @@ -19,7 +19,7 @@ public struct VCamBlendShapeAction: VCamAction { public var name: String { L10n.facialExpression.text } public var icon: Image { Image(systemName: "face.smiling") } - @UniAction(.setBlendShape) var setBlendShape + @UniAction(.currentBlendShape) var setBlendShape @MainActor public func callAsFunction(context: VCamActionContext) async throws { diff --git a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamLoadSceneAction.swift b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamLoadSceneAction.swift index 226d43b..c3694cd 100644 --- a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamLoadSceneAction.swift +++ b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamLoadSceneAction.swift @@ -19,10 +19,8 @@ public struct VCamLoadSceneAction: VCamAction { public var name: String { L10n.loadScene.text } public var icon: Image { Image(systemName: "square.3.stack.3d.top.fill") } - @UniAction(.loadScene) var loadScene - @MainActor public func callAsFunction(context: VCamActionContext) async throws { - loadScene(configuration.sceneId) + try? SceneManager.shared.loadScene(id: configuration.sceneId) } } diff --git a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMessageAction.swift b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMessageAction.swift index bd6cc81..4c9a167 100644 --- a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMessageAction.swift +++ b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMessageAction.swift @@ -19,7 +19,7 @@ public struct VCamMessageAction: VCamAction { public var name: String { L10n.message.text } public var icon: Image { Image(systemName: "text.bubble") } - @UniState(.message) private var message + @UniState(.message, name: "message") private var message @MainActor public func callAsFunction(context: VCamActionContext) async throws { diff --git a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMotionAction.swift b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMotionAction.swift index d05db09..5df5e5f 100644 --- a/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMotionAction.swift +++ b/app/xcode/Sources/VCamUI/Shortcut/Action/VCamMotionAction.swift @@ -8,6 +8,7 @@ import AppKit import VCamEntity import VCamLocalization +import VCamBridge import struct SwiftUI.Image public struct VCamMotionAction: VCamAction { @@ -19,10 +20,29 @@ public struct VCamMotionAction: VCamAction { public var name: String { L10n.motion.text } public var icon: Image { Image(systemName: "figure.wave") } - @UniAction(.triggerMotion) var triggerMotion - @MainActor public func callAsFunction(context: VCamActionContext) async throws { - triggerMotion(configuration.motion) + switch configuration.motion { + case .hi: + UniBridge.shared.motionHello() + case .bye: + UniBridge.shared.motionBye.wrappedValue.toggle() + case .jump: + UniBridge.shared.motionJump() + case .cheer: + UniBridge.shared.motionYear() + case .what: + UniBridge.shared.motionWhat() + case .pose: + UniBridge.shared.motionWin() + case .nod: + UniBridge.shared.motionNod.wrappedValue.toggle() + case .no: + UniBridge.shared.motionShakeHead.wrappedValue.toggle() + case .shudder: + UniBridge.shared.motionShakeBody.wrappedValue.toggle() + case .run: + UniBridge.shared.motionRun.wrappedValue.toggle() + } } } diff --git a/app/xcode/Sources/VCamUI/Shortcut/VCamShortcutBuilderView.swift b/app/xcode/Sources/VCamUI/Shortcut/VCamShortcutBuilderView.swift index 26dab32..5381a38 100644 --- a/app/xcode/Sources/VCamUI/Shortcut/VCamShortcutBuilderView.swift +++ b/app/xcode/Sources/VCamUI/Shortcut/VCamShortcutBuilderView.swift @@ -8,6 +8,7 @@ import SwiftUI import VCamEntity import VCamLocalization +import VCamBridge public struct VCamShortcutBuilderView: View { public init(shortcut: Binding) { @@ -140,9 +141,6 @@ struct VCamShortcutBuilderActionItemEditView: View { let shortcut: VCamShortcut @Binding var configuration: AnyVCamActionConfiguration - @UniState(.cachedBlendShapes) var cachedBlendShapes - @UniState(.scenes) var scenes - var body: some View { switch configuration { case let .emoji(configuration): @@ -152,13 +150,13 @@ struct VCamShortcutBuilderActionItemEditView: View { case let .motion(configuration): VCamActionEditorPicker(item: .init(configuration, keyPath: \.motion, to: $configuration), items: VCamAvatarMotion.allCases) case let .blendShape(configuration): - VCamActionEditorPicker(item: .init(configuration, keyPath: \.blendShape, to: $configuration), items: cachedBlendShapes) + VCamActionEditorPicker(item: .init(configuration, keyPath: \.blendShape, to: $configuration), items: UniBridge.shared.cachedBlendShapes) case let .wait(configuration): VCamActionEditorDurationField(value: .init(configuration, keyPath: \.duration, to: $configuration)) case .resetCamera: EmptyView() case let .loadScene(configuration): - VCamActionEditorPicker(item: .init(configuration, keyPath: \.sceneId, to: $configuration), items: scenes, mapValue: \.id) + VCamActionEditorPicker(item: .init(configuration, keyPath: \.sceneId, to: $configuration), items: SceneManager.shared.scenes, mapValue: \.id) case let .appleScript(configuration): VCamActionEditorCodeEditor(id: shortcut.id, actionId: configuration.id, name: VCamAppleScriptAction.scriptName) } diff --git a/app/xcode/Sources/VCamUI/UniAction.swift b/app/xcode/Sources/VCamUI/UniAction.swift index 353001f..328c8ce 100644 --- a/app/xcode/Sources/VCamUI/UniAction.swift +++ b/app/xcode/Sources/VCamUI/UniAction.swift @@ -7,6 +7,7 @@ import Foundation import VCamEntity +import VCamBridge @propertyWrapper public struct UniAction: Equatable { public init(action: @escaping (Arguments) -> Void) { @@ -39,40 +40,15 @@ public extension UniAction.Action { } public extension UniAction { - init(action: @escaping () -> Void) { - self.action = { _ in action() } + init(_ type: UniBridge.TriggerType) { + let mapper = UniBridge.shared.triggerMapper + self.init(action: mapper.trigger(type)) } } -// MARK: - Open source later - -public extension UniAction { - init(_ action: InternalUniAction) { - self.action = action.action - } -} - -public struct InternalUniAction { - public init(action: @escaping (Arguments) -> Void) { - self.action = action +public extension UniAction { + init(_ type: UniBridge.StringType) { + let mapper = UniBridge.shared.stringMapper + self.init(action: mapper.set(type)) } - - fileprivate let action: (Arguments) -> Void -} - -public extension InternalUniAction { - static var resetCamera = Self.init { _ in } -} - -public extension InternalUniAction { - static var showEmojiStamp = Self.init { _ in } - static var setBlendShape = Self.init { _ in } -} - -public extension InternalUniAction { - static var triggerMotion = Self.init { _ in } -} - -public extension InternalUniAction { - static var loadScene = Self.init { _ in } } diff --git a/app/xcode/Sources/VCamUI/UniState.swift b/app/xcode/Sources/VCamUI/UniState.swift index 59cd122..743f8d8 100644 --- a/app/xcode/Sources/VCamUI/UniState.swift +++ b/app/xcode/Sources/VCamUI/UniState.swift @@ -7,19 +7,13 @@ import SwiftUI import VCamEntity +import VCamBridge @propertyWrapper @dynamicMemberLookup public struct UniState: DynamicProperty { - public init(_ state: CustomState) { - get = state.get - set = state.set - name = state.name - reloadThrottle = state.reloadThrottle - } - - private let get: () -> Value - private let set: (Value) -> Void - private var name = "" - private var reloadThrottle = false + let get: () -> Value + let set: (Value) -> Void + var name = "" + var reloadThrottle = false @UniReload private var reload: Void @@ -45,38 +39,44 @@ import VCamEntity wrappedValue[keyPath: keyPath] = newValue.wrappedValue } } +} - public struct CustomState { - public init(get: @escaping () -> Value, set: @escaping (Value) -> Void, name: String = "", reloadThrottle: Bool = false) { - self.get = get - self.set = set - self.name = name - self.reloadThrottle = reloadThrottle - } +public extension UniState { + init(binding: Binding) { + self.init(get: { binding.wrappedValue }, set: { binding.wrappedValue = $0 }) + } - public var get: () -> Value - public var set: (Value) -> Void - public var name = "" - public var reloadThrottle = false + init(_ type: UniBridge.BoolType, name: String, reloadThrottle: Bool = false) where Value == Bool { + let mapper = UniBridge.shared.boolMapper + self.init(get: { mapper.get(type) }, set: mapper.set(type), name: name, reloadThrottle: reloadThrottle) } -} -public enum InternalUniState { - public static var reload: (String, Bool) -> Void = { _, _ in } + init(_ type: UniBridge.FloatType, name: String, reloadThrottle: Bool = false) where Value == CGFloat { + let mapper = UniBridge.shared.floatMapper + self.init(get: { mapper.get(type) }, set: mapper.set(type), name: name, reloadThrottle: reloadThrottle) + } - public static var message = UniState.CustomState(get: { "" }, set: { _ in }) - public static var cachedBlendShapes = UniState<[String]>.CustomState(get: { [] }, set: { _ in }) - public static var scenes = UniState<[VCamScene]>.CustomState(get: { [] }, set: { _ in }) -} + init(_ type: UniBridge.IntType, name: String, reloadThrottle: Bool = false) where Value == Int32 { + let mapper = UniBridge.shared.intMapper + self.init(get: { mapper.get(type) }, set: mapper.set(type), name: name, reloadThrottle: reloadThrottle) + } -public extension UniState.CustomState { - static var message: Self { InternalUniState.message } -} + init(_ type: UniBridge.StringType, name: String, reloadThrottle: Bool = false) where Value == String { + let mapper = UniBridge.shared.stringMapper + self.init(get: { mapper.get(type) }, set: mapper.set(type), name: name, reloadThrottle: reloadThrottle) + } -public extension UniState<[String]>.CustomState { - static var cachedBlendShapes: Self { InternalUniState.cachedBlendShapes } + init(_ type: UniBridge.ArrayType, name: String, as: Array.Type, reloadThrottle: Bool = false) where Value == Array { + let mapper = UniBridge.shared.arrayMapper + self.init(get: { mapper.binding(type, size: type.arraySize).wrappedValue }, set: mapper.set(type), name: name, reloadThrottle: reloadThrottle) + } + + init(_ type: UniBridge.StructType, name: String, as: Value.Type = Value.self, reloadThrottle: Bool = false) where Value: ValueBindingStructType { + let mapper = UniBridge.shared.structMapper + self.init(get: { mapper.binding(type).wrappedValue }, set: { mapper.binding(type).wrappedValue = $0 }, name: name, reloadThrottle: reloadThrottle) + } } -public extension UniState<[VCamScene]>.CustomState { - static var scenes: Self { InternalUniState.scenes } +public enum InternalUniState { + public static var reload: (String, Bool) -> Void = { _, _ in } } diff --git a/app/xcode/Sources/VCamUI/WebRenderer.swift b/app/xcode/Sources/VCamUI/WebRenderer.swift new file mode 100644 index 0000000..09fc8f6 --- /dev/null +++ b/app/xcode/Sources/VCamUI/WebRenderer.swift @@ -0,0 +1,356 @@ +// +// WebRenderer.swift +// +// +// Created by Tatsuya Tanaka on 2022/06/06. +// + +import Foundation +import CoreImage +import WebKit +import Combine +import SwiftUI +import VCamEntity + +public final class WebRenderer { + public let resource: Resource + public var size: CGSize + public var fps: Int + public var cropRect: CGRect = .init(x: 0, y: 0, width: 1, height: 1) + public var filter: ImageFilter? + + private let webView: WKWebView + private var timer: Timer? + private var render: ((CIImage) -> Void)? + + private var lastFrame = CIImage.empty() + + private var delegator: WebViewDelegator? + + private let viewHolder = NSWindow() + + public var css: String? { + didSet { + applyCurrentCSS() + } + } + + public var javaScript: String? { + didSet { + applyCurrentJavaScript() + } + } + + public let onFetchMetadata: ((VCamTagMetadata) -> Void)? + + public enum Resource: Equatable { + case url(URL) + case path(bookmark: Data) + + public var value: (url: URL?, bookmark: Data?) { + switch self { + case .url(let url): + return (url, nil) + case .path(let bookmark): + return (nil, bookmark) + } + } + + public var url: URL? { + switch self { + case .url(let url): + return url + case .path(let bookmark): + var isStale = false + return try? URL(resolvingBookmarkData: bookmark, options: .withSecurityScope, relativeTo: nil, bookmarkDataIsStale: &isStale) + } + } + } + + deinit { + stopRendering() + } + + public init(resource: Resource, size: CGSize, fps: Int, css: String?, js: String?, onFetchMetadata: ((VCamTagMetadata) -> Void)? = nil) { + self.resource = resource + self.size = size + self.fps = fps + self.css = css + self.javaScript = js + self.onFetchMetadata = onFetchMetadata + cropRect = Self.makeCropRect(with: size) + + let webView = Self.makeWebView(url: resource.url, size: size, window: viewHolder) + self.webView = webView + + render = { _ in } + timer = makeTimer() + + delegator = WebViewDelegator { [weak self] in + guard let self = self else { return } + self.applyCurrentCSS() + self.applyCurrentJavaScript() + self.webView.readMetadata(renderer: self) + self.renderWebViewTexture() + } + webView.navigationDelegate = delegator + } + + fileprivate func updateSize(width: Int, height: Int) { + size = .init(width: width, height: height) + webView.frame.size = size + viewHolder.contentView?.frame.size = size + cropRect = Self.makeCropRect(with: size) + } + + fileprivate func updateFps(_ fps: Int) { + if self.fps != fps { + self.fps = fps + timer = makeTimer() + } + } + + fileprivate func update(by metadata: VCamTagMetadata) { + if metadata.width != nil || metadata.height != nil { + size = .init(width: metadata.width ?? Int(size.width), height: metadata.height ?? Int(size.height)) + webView.frame.size = size + viewHolder.contentView?.frame.size = size + cropRect = Self.makeCropRect(with: size) + } + + if let fps = metadata.fps, self.fps != fps { + self.fps = fps + timer = makeTimer() + } + + onFetchMetadata?(metadata) + } + + private static func makeCropRect(with size: CGSize) -> CGRect { + if size.width < size.height { + return .init(origin: .zero, size: .init(width: size.width / size.height, height: 1)) + } else { + return .init(origin: .zero, size: .init(width: 1, height: size.height / size.width)) + } + } + + private static func makeWebView(url: URL?, size: CGSize, window: NSWindow) -> WKWebView { + let webView = WKWebView(frame: .init(origin: .zero, size: size)) + webView.customUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36" + webView.wantsLayer = true + webView.layer?.backgroundColor = .clear + webView.setValue(false, forKey: "drawsBackground") // https://developer.apple.com/forums/thread/121139 + if let url = url { + let request = URLRequest(url: url, cachePolicy: .reloadIgnoringLocalCacheData, timeoutInterval: 10) + webView.load(request) + } + + let containerView = NSView() + containerView.addSubview(webView) + containerView.frame.size = size + window.isReleasedWhenClosed = false + window.contentView = containerView // It seems that the YouTube Live chat's JS won't work unless it belongs to a window and is not hidden. + window.setFrame(.init(x: 0, y: 0, width: 1, height: 1), display: true, animate: false) // A size of more than 1pt is required to make it visible. + window.makeKeyAndOrderFront(nil) // If not visible, processes on view-in won't work & JS execution priority decreases, causing issues like the clock's second hand stuttering + return webView + } + + public static func snapshot(resource: Resource, size: CGSize, fps: Int, css: String?, js: String?, onFetchMetadata: ((VCamTagMetadata) -> Void)? = nil) -> (WebRenderer, AnyPublisher) { + let renderer = WebRenderer(resource: resource, size: size, fps: fps, css: css, js: js, onFetchMetadata: onFetchMetadata) + let publisher = PassthroughSubject() + + renderer.setRenderTexture { image in + publisher.send(image) + } + + return (renderer, publisher.eraseToAnyPublisher()) + } + + private func makeTimer() -> Timer? { + guard fps > 0 else { return nil } + return Timer.scheduledTimer(withTimeInterval: 1.0 / TimeInterval(fps), repeats: true) { [weak self] _ in + guard let self = self else { return } + self.renderWebViewTexture() + } + } + + private func renderWebViewTexture() { + webView.takeWebViewSnapshot(filter: self.filter) { raw, filtered in + self.lastFrame = raw + self.render?(filtered) + } + } + + private func applyCurrentCSS() { + guard let css = css, !css.isEmpty else { + return + } + webView.applyCSS(css) + } + + private func applyCurrentJavaScript() { + guard let javaScript = javaScript, !javaScript.isEmpty else { + return + } + webView.evaluateJavaScript(javaScript, completionHandler: nil) + } + + public func showWindow() { + let size = size + let webView = self.webView + let containerView = webView.superview + webView.removeFromSuperview() + + VCamWindow.showWindow(title: L10n.interact.text) { close in + ModalSheet(doneTitle: L10n.done.text, done: close) { + NSViewRepresentableBuilder { + webView + } + .frame(width: size.width, height: size.height) + } + } close: { [weak self] in + self?.renderWebViewTexture() + containerView?.addSubview(webView) + } + } + + final class WebViewDelegator: NSObject, WKNavigationDelegate { + init(didFinish: @escaping () -> Void) { + self.didFinish = didFinish + super.init() + } + + let didFinish: () -> Void + + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + didFinish() + } + } + + public struct VCamTagMetadata { + var width: Int? + var height: Int? + var fps: Int? + var hasChange: Bool { + width != nil || height != nil || fps != nil + } + } +} + +private extension WKWebView { + func takeWebViewSnapshot(filter: ImageFilter?, render: ((CIImage, CIImage) -> Void)?) { + takeSnapshot(with: nil) { image, error in + guard let rawImage = image?.ciImage else { + return + } + let filteredImage = filter?.apply(to: rawImage) ?? rawImage + render?(rawImage, filteredImage) + } + } +} + +struct NSViewRepresentableBuilder: NSViewRepresentable { + init(content: @escaping () -> Content, update: ((Content, Context) -> Void)? = nil) { + self.content = content + self.update = update + } + + let content: () -> Content + let update: ((Content, Context) -> Void)? + + func makeNSView(context: Context) -> Content { + content() + } + + func updateNSView(_ nsView: Content, context: Context) { + update?(nsView, context) + } + +} + +extension WebRenderer: RenderTextureRenderer { + public func setRenderTexture(updator: @escaping (CIImage) -> Void) { + render = updator + renderWebViewTexture() + } + + public func snapshot() -> CIImage { + lastFrame + } + + public func disableRenderTexture() { + render = nil + } + + public func pauseRendering() { + timer?.invalidate() + timer = nil + } + + public func resumeRendering() { + timer = makeTimer() + } + + public func stopRendering() { + render = nil + timer?.invalidate() + timer = nil + viewHolder.contentView = nil + viewHolder.close() + } +} + +private extension WKWebView { + func applyCSS(_ css: String) { + let jsString = """ +(function () { + let prevElement_My9jGxsf = document.getElementById('vcamcss_My9jGxsf'); + if (prevElement_My9jGxsf) { + document.head.removeChild(prevElement_My9jGxsf); + } + + var style_My9jGxsf = document.createElement('style'); + style_My9jGxsf.setAttribute('id', 'vcamcss_My9jGxsf'); + style_My9jGxsf.innerHTML = ` + \(css) + `; + document.head.appendChild(style_My9jGxsf); +}()); +""" + evaluateJavaScript(jsString) { aaa, error in +// uniDebugLog("WKWebView Error: \(error.debugDescription)") + } + } + + func readMetadata(renderer: WebRenderer) { + let jsString = """ +(function () { + const vcamMetas = document.getElementsByTagName('vcam-meta'); + if (vcamMetas.length == 0) { return null; } + const vcamMeta = vcamMetas[0]; + return { + 'width': vcamMeta.getAttribute('width'), + 'height': vcamMeta.getAttribute('height'), + 'fps': vcamMeta.getAttribute('fps') + }; +}()); +""" + evaluateJavaScript(jsString) { response, error in + guard let result = response as? [String: String?] else { + return + } + + let meta = WebRenderer.VCamTagMetadata( + width: result["width"]?.flatMap(Int.init), + height: result["height"]?.flatMap(Int.init), + fps: result["fps"]?.flatMap(Int.init) + ) + + guard meta.hasChange else { return } + + DispatchQueue.main.async { + renderer.update(by: meta) + } + } + } +}