From 8c0f2c37b96d48e3e174561597a69514bf6f8014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AC=E5=BD=A6=E8=B5=A4=E5=B1=8B=E5=85=88?= Date: Sat, 2 Nov 2024 03:28:01 +0900 Subject: [PATCH] Joint input actions draft impl --- plugin_Kinect360/Assets/Strings/en.json | 292 ++++++++++++++++-------- plugin_Kinect360/Kinect360.cs | 222 ++++++++++++++---- plugin_Kinect360/PackageUtils.cs | 50 ++++ 3 files changed, 424 insertions(+), 140 deletions(-) diff --git a/plugin_Kinect360/Assets/Strings/en.json b/plugin_Kinect360/Assets/Strings/en.json index a67dbea..8a9514a 100644 --- a/plugin_Kinect360/Assets/Strings/en.json +++ b/plugin_Kinect360/Assets/Strings/en.json @@ -1,97 +1,201 @@ { "language": "en", - "messages": [ - { - "id": "/Plugins/Kinect360/Statuses/Success", - "translation": "Success!\nS_OK\nEverything's good!" - }, - { - "id": "/Plugins/Kinect360/Statuses/Initializing", - "translation": "INITIALIZING\nS_NUI_INITIALIZING\nThe device is connected, but still initializing." - }, - { - "id": "/Plugins/Kinect360/Statuses/NotConnected", - "translation": "NOTCONNECTED\nE_NUI_NOTCONNECTED\nThe device is not connected." - }, - { - "id": "/Plugins/Kinect360/Statuses/NotGenuine", - "translation": "NOTGENUINE\nE_NUI_NOTGENUINE\nThe SDK is reporting the Kinect as invalid." - }, - { - "id": "/Plugins/Kinect360/Statuses/NotSupported", - "translation": "NOTSUPPORTED\nE_NUI_NOTSUPPORTED\nThe device is unsupported. (Xbox Kinect requires the SDK)" - }, - { - "id": "/Plugins/Kinect360/Statuses/InsufficientBandwidth", - "translation": "INSUFFICIENTBANDWIDTH\nE_NUI_INSUFFICIENTBANDWIDTH\nThe device is connected to a hub without the necessary bandwidth requirements." - }, - { - "id": "/Plugins/Kinect360/Statuses/NotPowered", - "translation": "NOTPOWERED\nE_NUI_NOTPOWERED\nThere is either a problem with your adapter/cables or with the Kinect device driver registration in Windows." - }, - { - "id": "/Plugins/Kinect360/Statuses/NotReady", - "translation": "NOTREADY\nE_NUI_NOTREADY\nThere was some other unspecified error." - }, - { - "id": "/Plugins/Kinect360/Settings/Labels/Angle", - "translation": "Kinect elevation angle:" - }, - { - "id": "/Plugins/Kinect360/Stages/Downloading/WiX", - "translation": "Downloading WiX Toolset..." - }, - { - "id": "/Plugins/Kinect360/Stages/Exceptions/WiX/Extraction", - "translation": "Toolset extraction failed! Exception: {0}" - }, - { - "id": "/Plugins/Kinect360/Stages/Exceptions/WiX/Installation", - "translation": "Toolset installation failed! Exception: {0}" - }, - { - "id": "/Plugins/Kinect360/Stages/Downloading/Runtime", - "translation": "Downloading Kinect for Xbox 360 Runtime..." - }, - { - "id": "/Plugins/Kinect360/Stages/Exceptions/Runtime/Installation", - "translation": "Runtime installation failed! Exception: {0}" - }, - { - "id": "/Plugins/Kinect360/Stages/Downloading/Toolkit", - "translation": "Downloading Kinect for Xbox 360 Toolkit..." - }, - { - "id": "/Plugins/Kinect360/Stages/Exceptions/Toolkit/Installation", - "translation": "Toolkit installation failed! Exception: {0}" - }, - { - "id": "/Plugins/Kinect360/Stages/Unpacking", - "translation": "Unpacking {0}..." - }, - { - "id": "/Plugins/Kinect360/Stages/Installing", - "translation": "Installing {0}..." - }, - { - "id": "/Plugins/Kinect360/Stages/Exceptions/Other", - "translation": "Exception: {0}" - }, - { - "id": "/Plugins/Kinect360/Stages/Dark/Error/Timeout", - "translation": "Failed to execute dark.exe in the allocated time!" - }, - { - "id": "/Plugins/Kinect360/Stages/Dark/Error/Result", - "translation": "Dark.exe exited with error code: {0}" - }, - { - "id": "/Plugins/Kinect360/Dependencies/Runtime/Name", - "translation": "Kinect for Xbox 360 Runtime" - }, - { - "id": "/Plugins/Kinect360/Dependencies/Toolkit/Name", - "translation": "Kinect for Xbox 360 Toolkit" - } - ] + "messages": [ + { + "id": "/Plugins/Kinect360/Statuses/Success", + "translation": "Success!\nS_OK\nEverything's good!" + }, + { + "id": "/Plugins/Kinect360/Statuses/Initializing", + "translation": "INITIALIZING\nS_NUI_INITIALIZING\nThe device is connected, but still initializing." + }, + { + "id": "/Plugins/Kinect360/Statuses/NotConnected", + "translation": "NOTCONNECTED\nE_NUI_NOTCONNECTED\nThe device is not connected." + }, + { + "id": "/Plugins/Kinect360/Statuses/NotGenuine", + "translation": "NOTGENUINE\nE_NUI_NOTGENUINE\nThe SDK is reporting the Kinect as invalid." + }, + { + "id": "/Plugins/Kinect360/Statuses/NotSupported", + "translation": "NOTSUPPORTED\nE_NUI_NOTSUPPORTED\nThe device is unsupported. (Xbox Kinect requires the SDK)" + }, + { + "id": "/Plugins/Kinect360/Statuses/InsufficientBandwidth", + "translation": "INSUFFICIENTBANDWIDTH\nE_NUI_INSUFFICIENTBANDWIDTH\nThe device is connected to a hub without the necessary bandwidth requirements." + }, + { + "id": "/Plugins/Kinect360/Statuses/NotPowered", + "translation": "NOTPOWERED\nE_NUI_NOTPOWERED\nThere is either a problem with your adapter/cables or with the Kinect device driver registration in Windows." + }, + { + "id": "/Plugins/Kinect360/Statuses/NotReady", + "translation": "NOTREADY\nE_NUI_NOTREADY\nThere was some other unspecified error." + }, + { + "id": "/Plugins/Kinect360/Settings/Labels/Angle", + "translation": "Kinect elevation angle:" + }, + { + "id": "/Plugins/Kinect360/Stages/Downloading/WiX", + "translation": "Downloading WiX Toolset..." + }, + { + "id": "/Plugins/Kinect360/Stages/Exceptions/WiX/Extraction", + "translation": "Toolset extraction failed! Exception: {0}" + }, + { + "id": "/Plugins/Kinect360/Stages/Exceptions/WiX/Installation", + "translation": "Toolset installation failed! Exception: {0}" + }, + { + "id": "/Plugins/Kinect360/Stages/Downloading/Runtime", + "translation": "Downloading Kinect for Xbox 360 Runtime..." + }, + { + "id": "/Plugins/Kinect360/Stages/Exceptions/Runtime/Installation", + "translation": "Runtime installation failed! Exception: {0}" + }, + { + "id": "/Plugins/Kinect360/Stages/Downloading/Toolkit", + "translation": "Downloading Kinect for Xbox 360 Toolkit..." + }, + { + "id": "/Plugins/Kinect360/Stages/Exceptions/Toolkit/Installation", + "translation": "Toolkit installation failed! Exception: {0}" + }, + { + "id": "/Plugins/Kinect360/Stages/Unpacking", + "translation": "Unpacking {0}..." + }, + { + "id": "/Plugins/Kinect360/Stages/Installing", + "translation": "Installing {0}..." + }, + { + "id": "/Plugins/Kinect360/Stages/Exceptions/Other", + "translation": "Exception: {0}" + }, + { + "id": "/Plugins/Kinect360/Stages/Dark/Error/Timeout", + "translation": "Failed to execute dark.exe in the allocated time!" + }, + { + "id": "/Plugins/Kinect360/Stages/Dark/Error/Result", + "translation": "Dark.exe exited with error code: {0}" + }, + { + "id": "/Plugins/Kinect360/Dependencies/Runtime/Name", + "translation": "Kinect for Xbox 360 Runtime" + }, + { + "id": "/Plugins/Kinect360/Dependencies/Toolkit/Name", + "translation": "Kinect for Xbox 360 Toolkit" + }, + { + "id": "/JointsEnum/JointHead", + "translation": "Head" + }, + { + "id": "/JointsEnum/JointNeck", + "translation": "Neck" + }, + { + "id": "/JointsEnum/JointSpineShoulder", + "translation": "Spine (Shoulders)" + }, + { + "id": "/JointsEnum/JointShoulderLeft", + "translation": "Left Shoulder" + }, + { + "id": "/JointsEnum/JointElbowLeft", + "translation": "Left Elbow" + }, + { + "id": "/JointsEnum/JointWristLeft", + "translation": "Left Wrist" + }, + { + "id": "/JointsEnum/JointHandLeft", + "translation": "Left Hand" + }, + { + "id": "/JointsEnum/JointHandTipLeft", + "translation": "Left Hand Tip" + }, + { + "id": "/JointsEnum/JointThumbLeft", + "translation": "Left Thumb" + }, + { + "id": "/JointsEnum/JointShoulderRight", + "translation": "Right Shoulder" + }, + { + "id": "/JointsEnum/JointElbowRight", + "translation": "Right Elbow" + }, + { + "id": "/JointsEnum/JointWristRight", + "translation": "Right Wrist" + }, + { + "id": "/JointsEnum/JointHandRight", + "translation": "Right Hand" + }, + { + "id": "/JointsEnum/JointHandTipRight", + "translation": "Right Hand Tip" + }, + { + "id": "/JointsEnum/JointThumbRight", + "translation": "Right Thumb" + }, + { + "id": "/JointsEnum/JointSpineMiddle", + "translation": "Spine (Middle)" + }, + { + "id": "/JointsEnum/JointSpineWaist", + "translation": "Waist" + }, + { + "id": "/JointsEnum/JointHipLeft", + "translation": "Left Hip" + }, + { + "id": "/JointsEnum/JointKneeLeft", + "translation": "Left Knee" + }, + { + "id": "/JointsEnum/JointFootLeft", + "translation": "Left Foot" + }, + { + "id": "/JointsEnum/JointFootTipLeft", + "translation": "Left Foot Tip" + }, + { + "id": "/JointsEnum/JointHipRight", + "translation": "Right Hip" + }, + { + "id": "/JointsEnum/JointKneeRight", + "translation": "Right Knee" + }, + { + "id": "/JointsEnum/JointFootRight", + "translation": "Right Foot" + }, + { + "id": "/JointsEnum/JointFootTipRight", + "translation": "Right Foot Tip" + }, + { + "id": "/JointsEnum/JointManual", + "translation": "Manual" + } + ] } \ No newline at end of file diff --git a/plugin_Kinect360/Kinect360.cs b/plugin_Kinect360/Kinect360.cs index dba7469..ab054ca 100644 --- a/plugin_Kinect360/Kinect360.cs +++ b/plugin_Kinect360/Kinect360.cs @@ -49,7 +49,13 @@ public class Kinect360 : KinectHandler.KinectHandler, ITrackingDevice public bool IsFlipSupported => true; public bool IsAppOrientationSupported => true; public object SettingsInterfaceRoot => InterfaceRoot; - private static IAmethystHost HostStatic { get; set; } + public static IAmethystHost HostStatic { get; set; } + + private readonly GestureDetector + _pauseDetectorLeft = new(), + _pauseDetectorRight = new(), + _pointDetectorLeft = new(), + _pointDetectorRight = new(); public ObservableCollection TrackedJoints { get; } = // Prepend all supported joints to the joints list @@ -72,12 +78,12 @@ TrackedJointType.JointHandTipLeft and not TrackedJointType.JointHandTipRight and { Name = "Left Point", Description = "Left hand point gesture", Guid = "8D83B89D-5FBD-4D52-B626-4E90BDD26B08", GetHost = () => HostStatic - }, - new KeyInputAction - { - Name = "Left Press", Description = "Left hand press gesture", - Guid = "E383258F-5918-4F1C-BC66-7325DB1F07E8", GetHost = () => HostStatic } + //new KeyInputAction + //{ + // Name = "Left Grab", Description = "Left hand grab gesture", + // Guid = "E383258F-5918-4F1C-BC66-7325DB1F07E8", GetHost = () => HostStatic + //} ], TrackedJointType.JointHandRight => [ @@ -90,12 +96,12 @@ TrackedJointType.JointHandTipLeft and not TrackedJointType.JointHandTipRight and { Name = "Right Point", Description = "Right hand point gesture", Guid = "C58EBCFE-0DF5-40FD-ABC1-06B415FA51BE", GetHost = () => HostStatic - }, - new KeyInputAction - { - Name = "Right Press", Description = "Right hand press gesture", - Guid = "801336BE-5BD5-4881-A390-D57D958592EF", GetHost = () => HostStatic } + //new KeyInputAction + //{ + // Name = "Right Grab", Description = "Right hand grab gesture", + // Guid = "801336BE-5BD5-4881-A390-D57D958592EF", GetHost = () => HostStatic + //} ], _ => [] } @@ -158,30 +164,6 @@ public void OnLoad() VerticalAlignment = VerticalAlignment.Center }; - // TODO TEMPORARY - var actionButtons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 6, Margin = new Thickness(5) }; - foreach (var action in TrackedJoints.SelectMany(x => x.SupportedInputActions)) - { - var button = new Button { Content = action.Name }; - actionButtons.Children.Add(button); - - button.Click += (_, _) => - { - try - { - if (action.IsUsed) - //action.Invoke(true); - Host.ReceiveKeyInput(action, true); - } - catch (Exception ex) - { - Host?.Log(ex); - Debugger.Break(); - } - }; - } - // TODO TEMPORARY - CameraImage = new WriteableBitmap(CameraImageWidth, CameraImageHeight); InterfaceRoot = new Page { @@ -193,14 +175,26 @@ public void OnLoad() { Orientation = Orientation.Horizontal, Children = { TiltTextBlock, TiltNumberBox } - }, - // TODO TEMPORARY - actionButtons - // TODO TEMPORARY + } } } }; + try + { + // Re-generate joint names + lock (Host.UpdateThreadLock) + { + for (var i = 0; i < TrackedJoints.Count; i++) + TrackedJoints[i] = TrackedJoints[i].WithName(Host?.RequestLocalizedString( + $"/JointsEnum/{TrackedJoints[i].Role.ToString()}") ?? TrackedJoints[i].Role.ToString()); + } + } + catch (Exception e) + { + Host?.Log($"Error setting joint names! Message: {e.Message}", LogSeverity.Error); + } + PluginLoaded = true; // Mark as already-loaded } @@ -256,14 +250,79 @@ public void Update() }); // Update camera feed - if (!IsCameraEnabled) return; - CameraImage.DispatcherQueue.TryEnqueue(async () => + if (IsCameraEnabled) + CameraImage.DispatcherQueue.TryEnqueue(async () => + { + var buffer = GetImageBuffer(); // Read from Kinect + if (buffer is null || buffer.Length <= 0) return; + await CameraImage.PixelBuffer.AsStream().WriteAsync(buffer); + CameraImage.Invalidate(); // Enqueue for preview refresh + }); + + // Update gestures + if (trackedJoints.Count != 20) return; + + try { - var buffer = GetImageBuffer(); // Read from Kinect - if (buffer is null || buffer.Length <= 0) return; - await CameraImage.PixelBuffer.AsStream().WriteAsync(buffer); - CameraImage.Invalidate(); // Enqueue for preview refresh - }); + var shoulderLeft = trackedJoints[(int)NativeKinectJoints.JointShoulderLeft].Position; + var shoulderRight = trackedJoints[(int)NativeKinectJoints.JointShoulderRight].Position; + var elbowLeft = trackedJoints[(int)NativeKinectJoints.JointElbowLeft].Position; + var elbowRight = trackedJoints[(int)NativeKinectJoints.JointElbowRight].Position; + var handLeft = trackedJoints[(int)NativeKinectJoints.JointWristLeft].Position; + var handRight = trackedJoints[(int)NativeKinectJoints.JointWristRight].Position; + + // >0.9f when elbow is not bent and the arm is straight : LEFT + var armDotLeft = Vector3.Dot( + Vector3.Normalize(elbowLeft - shoulderLeft), + Vector3.Normalize(handLeft - elbowLeft)); + + // >0.9f when the arm is pointing down : LEFT + var armDownDotLeft = Vector3.Dot( + new Vector3(0.0f, -1.0f, 0.0f), + Vector3.Normalize(handLeft - elbowLeft)); + + // >0.4f <0.6f when the arm is slightly tilted sideways : RIGHT + var armTiltDotLeft = Vector3.Dot( + Vector3.Normalize(shoulderLeft - shoulderRight), + Vector3.Normalize(handLeft - elbowLeft)); + + // >0.9f when elbow is not bent and the arm is straight : LEFT + var armDotRight = Vector3.Dot( + Vector3.Normalize(elbowRight - shoulderRight), + Vector3.Normalize(handRight - elbowRight)); + + // >0.9f when the arm is pointing down : RIGHT + var armDownDotRight = Vector3.Dot( + new Vector3(0.0f, -1.0f, 0.0f), + Vector3.Normalize(handRight - elbowRight)); + + // >0.4f <0.6f when the arm is slightly tilted sideways : RIGHT + var armTiltDotRight = Vector3.Dot( + Vector3.Normalize(shoulderRight - shoulderLeft), + Vector3.Normalize(handRight - elbowRight)); + + /* Trigger the detected gestures */ + + if (TrackedJoints[(int)NativeKinectJoints.JointHandLeft].SupportedInputActions.IsUsed(0, out var pauseActionLeft)) + Host.ReceiveKeyInput(pauseActionLeft, _pauseDetectorLeft.Update(armDotLeft > 0.9f && armTiltDotLeft is > 0.4f and < 0.7f)); + + if (TrackedJoints[(int)NativeKinectJoints.JointHandRight].SupportedInputActions.IsUsed(0, out var pauseActionRight)) + Host.ReceiveKeyInput(pauseActionRight, _pauseDetectorRight.Update(armDotRight > 0.9f && armTiltDotRight is > 0.4f and < 0.7f)); + + if (TrackedJoints[(int)NativeKinectJoints.JointHandLeft].SupportedInputActions.IsUsed(1, out var pointActionLeft)) + Host.ReceiveKeyInput(pointActionLeft, _pointDetectorLeft + .Update(armDotLeft > 0.9f && armTiltDotLeft is > -0.5f and < 0.5f && armDownDotLeft is > -0.3f and < 0.7f)); + + if (TrackedJoints[(int)NativeKinectJoints.JointHandRight].SupportedInputActions.IsUsed(1, out var pointActionRight)) + Host.ReceiveKeyInput(pointActionRight, _pointDetectorRight + .Update(armDotRight > 0.9f && armTiltDotRight is > -0.5f and < 0.5f && armDownDotRight is > -0.3f and < 0.7f)); + + /* Trigger the detected gestures */ + } + catch (Exception ex) + { + Host?.Log(ex); + } } public void SignalJoint(int jointId) @@ -306,4 +365,75 @@ public static Vector3 Safe(this Vector3 v) ? Vector3.Zero // Return a placeholder position vector : v; // If everything is fine, return the actual orientation } + + public static T At(this SortedSet set, int at) + { + return set.ElementAt(at); + } + + public static bool At(this SortedSet set, int at, out T result) + { + try + { + result = set.ElementAt(at); + } + catch + { + result = default; + return false; + } + + return true; + } + + public static bool IsUsed(this SortedSet set, int at) + { + return set.At(at, out var action) && (Kinect360.HostStatic?.CheckInputActionIsUsed(action) ?? false); + } + + public static bool IsUsed(this SortedSet set, int at, out IKeyInputAction action) + { + return set.At(at, out action) && (Kinect360.HostStatic?.CheckInputActionIsUsed(action) ?? false); + } + + public static TrackedJoint WithName(this TrackedJoint joint, string name) + { + return new TrackedJoint + { + Name = name, + Role = joint.Role, + Acceleration = joint.Acceleration, + AngularAcceleration = joint.AngularAcceleration, + AngularVelocity = joint.AngularVelocity, + Orientation = joint.Orientation, + Position = joint.Position, + SupportedInputActions = joint.SupportedInputActions, + TrackingState = joint.TrackingState, + Velocity = joint.Velocity + }; + } +} + +internal enum NativeKinectJoints +{ + JointHead, + JointSpineShoulder, + JointShoulderLeft, + JointElbowLeft, + JointWristLeft, + JointHandLeft, + JointShoulderRight, + JointElbowRight, + JointWristRight, + JointHandRight, + JointSpineMiddle, + JointSpineWaist, + JointHipLeft, + JointKneeLeft, + JointFootLeft, + JointFootTipLeft, + JointHipRight, + JointKneeRight, + JointFootRight, + JointFootTipRight } \ No newline at end of file diff --git a/plugin_Kinect360/PackageUtils.cs b/plugin_Kinect360/PackageUtils.cs index 68aff6c..f13ba79 100644 --- a/plugin_Kinect360/PackageUtils.cs +++ b/plugin_Kinect360/PackageUtils.cs @@ -1,7 +1,9 @@ using System; +using System.Diagnostics; using System.IO; using Windows.ApplicationModel; using Windows.Storage; +using Amethyst.Plugins.Contract; namespace plugin_Kinect360; @@ -45,4 +47,52 @@ public static void CopyToFolder(this DirectoryInfo source, string destination, b foreach (var newPath in source.GetFiles("*.*", SearchOption.AllDirectories)) newPath.CopyTo(newPath.FullName.Replace(source.FullName, destination), true); } +} + +public class GestureDetector +{ + private bool Value { get; set; } + private bool ValueBlock { get; set; } + private Stopwatch Timer { get; set; } = new(); + + public bool Update(bool value) + { + // ReSharper disable once ConvertIfStatementToSwitchStatement + if (!Value && value) + { + //Console.WriteLine("Restarting gesture timer..."); + ValueBlock = false; + Timer.Restart(); + Value = true; + return false; + } + + if (!Value && !value) + { + //Console.WriteLine("Resetting gesture timer..."); + ValueBlock = false; + Timer.Reset(); + return false; + } + + Value = value; + + switch (Timer.ElapsedMilliseconds) + { + case >= 1000 when !ValueBlock: + //Console.Write("Gesture detected! "); + Kinect360.HostStatic?.PlayAppSound(SoundType.Focus); + ValueBlock = true; + return true; + case >= 3000 when ValueBlock: + //Console.Write("Restarting timer..."); + Kinect360.HostStatic?.PlayAppSound(SoundType.Focus); + ValueBlock = false; + Timer.Restart(); + return true; + default: + //Console.WriteLine("Gesture detected! Waiting for the timer..."); + return false; + } + } } \ No newline at end of file