From 12ea3ffd625e1b8421e0e137b781755f30314577 Mon Sep 17 00:00:00 2001 From: Manav Gagvani Date: Thu, 18 Jul 2024 11:13:31 -0400 Subject: [PATCH] Add support for video game playing - both data recording and inference --- donkeycar/parts/actuator.py | 31 ++++++++++++++++ donkeycar/parts/camera.py | 57 +++++++++++++++++++++++++++++ donkeycar/templates/cfg_complete.py | 3 +- donkeycar/templates/complete.py | 20 ++++++++++ 4 files changed, 110 insertions(+), 1 deletion(-) diff --git a/donkeycar/parts/actuator.py b/donkeycar/parts/actuator.py index 848feebc3..401441cd4 100644 --- a/donkeycar/parts/actuator.py +++ b/donkeycar/parts/actuator.py @@ -21,6 +21,11 @@ logger.warning(f"RPi.GPIO was not imported. {e}") globals()["GPIO"] = None +try: + import pyxinput +except ImportError as e: + logger.warn(f"pyxinput was not imported. {e}") + from donkeycar.parts.pins import OutputPin, PwmPin, PinState from donkeycar.utilities.deprecated import deprecated @@ -1158,3 +1163,29 @@ def shutdown(self): # stop vehicle self.run(0) self.running = False + +class VirtualController: + ''' + Simulate a controller with a virtual joystick. + For use with video game control. + ''' + def __init__(self): + self.angle = 0 + self.throttle = 0 + self.controller = pyxinput.vController() + + def run(self, angle, throttle): + self.angle = angle + self.throttle = throttle + + # angle + self.controller.set_value('AxisLx', angle) + + # throttle + if throttle > 0: + self.controller.set_value('TriggerR', throttle) + else: + self.controller.set_value('TriggerL', -throttle) + + def shutdown(self): + self.controller.UnPlug() diff --git a/donkeycar/parts/camera.py b/donkeycar/parts/camera.py index 1377089c7..70f52202c 100644 --- a/donkeycar/parts/camera.py +++ b/donkeycar/parts/camera.py @@ -1,8 +1,10 @@ import logging import os import time +import threading import numpy as np from PIL import Image +import mss import glob from donkeycar.utils import rgb2gray @@ -329,6 +331,61 @@ def shutdown(self): self.running = False time.sleep(0.5) +class ScreenCamera(BaseCamera): + ''' + Camera that uses mss to capture the screen + For capturing video games + ''' + + def __init__(self, image_w=160, image_h=120, image_d=3, + vflip=False, hflip=False): + self.image_w = image_w + self.image_h = image_h + self.image_d = image_d + self.vflip = vflip + self.hflip = hflip + self.sct = mss.mss() + self.running = True + + self._monitor_thread = threading.Thread(target=self.take_screenshot, args=()) + self._monitor_thread.daemon = True + self._monitor_thread.start() + + def take_screenshot(self): + # Capture the screen + monitor = {"top": 0, + "left": 320, + "width": 1920, + "height": 1080 + } + sct_img = self.sct.grab(monitor) + img = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX') + img = img.resize((self.image_w, self.image_h)) + img_arr = np.asarray(img) + if self.vflip: + img_arr = np.flipud(img_arr) + if self.hflip: + img_arr = np.fliplr(img_arr) + self.frame = img_arr + return img + + def update(self): + if self.running: + img = self.take_screenshot() + return img + + def run(self): + self.update() + assert self.frame is not None + return self.frame + + def run_threaded(self): + return self.frame + + def shutdown(self): + self.running = False + self.sct.close() + class MockCamera(BaseCamera): ''' diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index d468f6b3c..f2dee6855 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -25,7 +25,7 @@ MAX_LOOPS = None # the vehicle loop can abort after this many iterations, when given a positive integer. #CAMERA -CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST) +CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST|SCREENSHOT) IMAGE_W = 160 IMAGE_H = 120 IMAGE_DEPTH = 3 # default RGB=3, make 1 for mono @@ -66,6 +66,7 @@ # "DC_TWO_WHEEL_L298N" using HBridge in 3-pin mode to control two drive motors, one of the left and one on the right. # "MOCK" no drive train. This can be used to test other features in a test rig. # "VESC" VESC Motor controller to set servo angle and duty cycle +# "VIRTUAL" Simulates an Xbox controller, can be used with Donkey in a video game environment. # (deprecated) "SERVO_HBRIDGE_PWM" use ServoBlaster to output pwm control from the PiZero directly to control steering, # and HBridge for a drive motor. # (deprecated) "PIGPIO_PWM" uses Raspberrys internal PWM diff --git a/donkeycar/templates/complete.py b/donkeycar/templates/complete.py index 32c4a7168..028144fb6 100644 --- a/donkeycar/templates/complete.py +++ b/donkeycar/templates/complete.py @@ -820,6 +820,9 @@ def get_camera(cfg): elif cfg.CAMERA_TYPE == "MOCK": from donkeycar.parts.camera import MockCamera cam = MockCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH) + elif cfg.CAMERA_TYPE == "SCREENSHOT": + from donkeycar.parts.camera import ScreenCamera + cam = ScreenCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH) else: raise(Exception("Unkown camera type: %s" % cfg.CAMERA_TYPE)) return cam @@ -873,6 +876,18 @@ def add_camera(V, cfg, camera_type): 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'], threaded=True) + + elif cfg.CAMERA_TYPE == "SCREENSHOT": + from donkeycar.parts.camera import ScreenCamera + cam = ScreenCamera( + image_w=cfg.IMAGE_W, + image_h=cfg.IMAGE_H, + vflip=cfg.CAMERA_VFLIP, + hflip=cfg.CAMERA_HFLIP) + V.add(cam, + outputs=['cam/image_array'], + threaded=False) + else: inputs = [] outputs = ['cam/image_array'] @@ -1131,6 +1146,11 @@ def add_drivetrain(V, cfg): ) V.add(vesc, inputs=['steering', 'throttle']) + elif cfg.DRIVE_TRAIN_TYPE == "VIRTUAL": + from donkeycar.parts.actuator import VirtualController + logger.info("Creating virtual controller") + V.add(VirtualController(), inputs=['steering', 'throttle']) + if __name__ == '__main__': args = docopt(__doc__)