Add Support for Video Games as a Medium of Control/Recording

donkeycar/parts/actuator.py

@@ -21,6 +21,11 @@
     logger.warning(f"RPi.GPIO was not imported. {e}")
     globals()["GPIO"] = None
 
+try:
+    import pyxinput
+except ImportError as e:
+    logger.warn(f"pyxinput was not imported. {e}")
+
 from donkeycar.parts.pins import OutputPin, PwmPin, PinState
 from donkeycar.utilities.deprecated import deprecated
 
@@ -1158,3 +1163,29 @@ def shutdown(self):
         # stop vehicle
         self.run(0)
         self.running = False
+
+class VirtualController:
+    '''
+    Simulate a controller with a virtual joystick.
+    For use with video game control.
+    '''
+    def __init__(self):
+        self.angle = 0
+        self.throttle = 0
+        self.controller = pyxinput.vController()
+
+    def run(self, angle, throttle):
+        self.angle = angle
+        self.throttle = throttle
+
+        # angle
+        self.controller.set_value('AxisLx', angle)
+
+        # throttle
+        if throttle > 0:
+            self.controller.set_value('TriggerR', throttle)
+        else:
+            self.controller.set_value('TriggerL', -throttle)
+
+    def shutdown(self):
+        self.controller.UnPlug()

donkeycar/parts/camera.py

@@ -1,8 +1,10 @@
 import logging
 import os
 import time
+import threading
 import numpy as np
 from PIL import Image
+import mss
 import glob
 from donkeycar.utils import rgb2gray
 
@@ -329,6 +331,61 @@ def shutdown(self):
         self.running = False
         time.sleep(0.5)
 
+class ScreenCamera(BaseCamera):
+    '''
+    Camera that uses mss to capture the screen
+    For capturing video games
+    '''
+
+    def __init__(self, image_w=160, image_h=120, image_d=3,
+                 vflip=False, hflip=False):
+        self.image_w = image_w
+        self.image_h = image_h
+        self.image_d = image_d
+        self.vflip = vflip
+        self.hflip = hflip
+        self.sct = mss.mss()
+        self.running = True
+
+        self._monitor_thread = threading.Thread(target=self.take_screenshot, args=())
+        self._monitor_thread.daemon = True
+        self._monitor_thread.start()
+
+    def take_screenshot(self):
+        # Capture the screen
+        monitor = {"top": 0, 
+                   "left": 320, 
+                   "width": 1920, 
+                   "height": 1080
+                   }
+        sct_img = self.sct.grab(monitor)
+        img = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX')
+        img = img.resize((self.image_w, self.image_h))
+        img_arr = np.asarray(img)
+        if self.vflip:
+            img_arr = np.flipud(img_arr)
+        if self.hflip:
+            img_arr = np.fliplr(img_arr)
+        self.frame = img_arr
+        return img
+
+    def update(self):
+        if self.running:
+            img = self.take_screenshot()
+            return img
+
+    def run(self):
+        self.update()
+        assert self.frame is not None
+        return self.frame
+
+    def run_threaded(self):
+        return self.frame
+
+    def shutdown(self):
+        self.running = False
+        self.sct.close()
+
 
 class MockCamera(BaseCamera):
     '''

donkeycar/templates/cfg_complete.py

@@ -25,7 +25,7 @@
 MAX_LOOPS = None        # the vehicle loop can abort after this many iterations, when given a positive integer.
 
 #CAMERA
-CAMERA_TYPE = "PICAM"   # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST)
+CAMERA_TYPE = "PICAM"   # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST|SCREENSHOT)
 IMAGE_W = 160
 IMAGE_H = 120
 IMAGE_DEPTH = 3         # default RGB=3, make 1 for mono
@@ -66,6 +66,7 @@
 # "DC_TWO_WHEEL_L298N" using HBridge in 3-pin mode to control two drive motors, one of the left and one on the right.
 # "MOCK" no drive train.  This can be used to test other features in a test rig.
 # "VESC" VESC Motor controller to set servo angle and duty cycle
+# "VIRTUAL" Simulates an Xbox controller, can be used with Donkey in a video game environment.
 # (deprecated) "SERVO_HBRIDGE_PWM" use ServoBlaster to output pwm control from the PiZero directly to control steering,
 #                                  and HBridge for a drive motor.
 # (deprecated) "PIGPIO_PWM" uses Raspberrys internal PWM

donkeycar/templates/complete.py

@@ -820,6 +820,9 @@ def get_camera(cfg):
         elif cfg.CAMERA_TYPE == "MOCK":
             from donkeycar.parts.camera import MockCamera
             cam = MockCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH)
+        elif cfg.CAMERA_TYPE == "SCREENSHOT":
+            from donkeycar.parts.camera import ScreenCamera
+            cam = ScreenCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH)
         else:
             raise(Exception("Unkown camera type: %s" % cfg.CAMERA_TYPE))
     return cam
@@ -873,6 +876,18 @@ def add_camera(V, cfg, camera_type):
                        'imu/acl_x', 'imu/acl_y', 'imu/acl_z',
                        'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'],
               threaded=True)
+
+    elif cfg.CAMERA_TYPE == "SCREENSHOT":
+        from donkeycar.parts.camera import ScreenCamera
+        cam = ScreenCamera(
+            image_w=cfg.IMAGE_W, 
+            image_h=cfg.IMAGE_H,
+            vflip=cfg.CAMERA_VFLIP,
+            hflip=cfg.CAMERA_HFLIP)
+        V.add(cam, 
+              outputs=['cam/image_array'], 
+              threaded=False)
+
     else:
         inputs = []
         outputs = ['cam/image_array']
@@ -1131,6 +1146,11 @@ def add_drivetrain(V, cfg):
                         )
             V.add(vesc, inputs=['steering', 'throttle'])
 
+        elif cfg.DRIVE_TRAIN_TYPE == "VIRTUAL":
+            from donkeycar.parts.actuator import VirtualController
+            logger.info("Creating virtual controller")
+            V.add(VirtualController(), inputs=['steering', 'throttle'])
+
 
 if __name__ == '__main__':
     args = docopt(__doc__)