Switch from package:opencv_ffi to opencv_dart (#15)

While the `opencv_ffi` package was originally made specifically for
rover, it has since been superseded by the more feature-complete
`opencv_dart` package.

> [!important]
> This package uses Native Assets, which aren't officially stable yet.
To run, [install conan](https://docs.conan.io/2/installation.html) and
run with
> ```bash
> dart --enable-experiment=native-assets run
> ```

---------

Co-authored-by: Levi Lesches <[email protected]>

lib/realsense.dart

@@ -0,0 +1,2 @@
+export "src/realsense/interface.dart";
+export "src/realsense/ffi.dart";

lib/src/collection.dart

@@ -1,42 +1,9 @@
 import "dart:async";
-import "dart:io";
 
 import "package:burt_network/burt_network.dart";
-import "package:opencv_ffi/opencv_ffi.dart";
 
 import "package:video/video.dart";
 
-/// Default details for a camera
-///
-/// Used when first creating the camera objects
-CameraDetails getDefaultDetails(CameraName name) => CameraDetails(
-  name: name,
-  resolutionWidth: 600,
-  resolutionHeight: 600,
-  quality: 75,
-  fps: 24,
-  status: CameraStatus.CAMERA_ENABLED,
-);
-
-/// Default details for the RealSense camera.
-///
-/// These settings are balanced between autonomy depth and normal RGB.
-CameraDetails getRealsenseDetails(CameraName name) => CameraDetails(
-  name: name,
-  resolutionWidth: 300,
-  resolutionHeight: 300,
-  quality: 50,
-  fps: 0,
-  status: CameraStatus.CAMERA_ENABLED,
-);
-
-/// Returns the camera depending on device program is running
-///
-/// Uses [cameraNames] or [cameraIndexes]
-Camera getCamera(CameraName name) => Platform.isWindows
-  ? Camera.fromIndex(cameraIndexes[name]!)
-  : Camera.fromName(cameraNames[name]!);
-
 /// Class to contain all video devices
 class Collection extends Service {
   /// The [RoverSocket] to send messages through

lib/src/generated/librealsense_ffi_bindings.dart

@@ -49,17 +49,18 @@ class LibRealSenseBindings {
   late final _RealSense_free = _RealSense_freePtr.asFunction<
       void Function(ffi.Pointer<NativeRealSense>)>();
 
-  int RealSense_init(
+  BurtRsStatus RealSense_init(
     ffi.Pointer<NativeRealSense> ptr,
   ) {
-    return _RealSense_init(
+    return BurtRsStatus.fromValue(_RealSense_init(
       ptr,
-    );
+    ));
   }
 
   late final _RealSense_initPtr = _lookup<
-          ffi.NativeFunction<ffi.Int32 Function(ffi.Pointer<NativeRealSense>)>>(
-      'RealSense_init');
+      ffi.NativeFunction<
+          ffi.UnsignedInt Function(
+              ffi.Pointer<NativeRealSense>)>>('RealSense_init');
   late final _RealSense_init = _RealSense_initPtr.asFunction<
       int Function(ffi.Pointer<NativeRealSense>)>();
 
@@ -94,17 +95,18 @@ class LibRealSenseBindings {
       .asFunction<BurtRsConfig Function(ffi.Pointer<NativeRealSense>)>();
 
   /// Streams
-  int RealSense_startStream(
+  BurtRsStatus RealSense_startStream(
     ffi.Pointer<NativeRealSense> ptr,
   ) {
-    return _RealSense_startStream(
+    return BurtRsStatus.fromValue(_RealSense_startStream(
       ptr,
-    );
+    ));
   }
 
   late final _RealSense_startStreamPtr = _lookup<
-          ffi.NativeFunction<ffi.Int32 Function(ffi.Pointer<NativeRealSense>)>>(
-      'RealSense_startStream');
+      ffi.NativeFunction<
+          ffi.UnsignedInt Function(
+              ffi.Pointer<NativeRealSense>)>>('RealSense_startStream');
   late final _RealSense_startStream = _RealSense_startStreamPtr.asFunction<
       int Function(ffi.Pointer<NativeRealSense>)>();
 
@@ -153,12 +155,24 @@ class LibRealSenseBindings {
       void Function(ffi.Pointer<NativeFrames>)>();
 }
 
-abstract class BurtRsStatus {
-  static const int BurtRsStatus_ok = 0;
-  static const int BurtRsStatus_no_device = 1;
-  static const int BurtRsStatus_too_many_devices = 2;
-  static const int BurtRsStatus_resolution_unknown = 3;
-  static const int BurtRsStatus_scale_unknown = 4;
+enum BurtRsStatus {
+  BurtRsStatus_ok(0),
+  BurtRsStatus_no_device(1),
+  BurtRsStatus_too_many_devices(2),
+  BurtRsStatus_resolution_unknown(3),
+  BurtRsStatus_scale_unknown(4);
+
+  final int value;
+  const BurtRsStatus(this.value);
+
+  static BurtRsStatus fromValue(int value) => switch (value) {
+        0 => BurtRsStatus_ok,
+        1 => BurtRsStatus_no_device,
+        2 => BurtRsStatus_too_many_devices,
+        3 => BurtRsStatus_resolution_unknown,
+        4 => BurtRsStatus_scale_unknown,
+        _ => throw ArgumentError("Unknown value for BurtRsStatus: $value"),
+      };
 }
 
 final class BurtRsConfig extends ffi.Struct {

lib/src/isolates/child.dart

@@ -1,15 +1,29 @@
 import "dart:async";
+import "dart:typed_data";
 
 import "package:burt_network/burt_network.dart";
 import "package:typed_isolate/typed_isolate.dart";
-import "package:opencv_ffi/opencv_ffi.dart";
-
 import "package:video/video.dart";
 
 /// The maximum size of a UDP packet, in bytes (minus a few to be safe).
 const maxPacketLength = 60000;
 
 /// A child isolate that manages a single camera and streams frames from it.
+///
+/// This class can represent any combination of hardware and software, such as regular USB cameras
+/// driven by OpenCV or a depth camera read with the RealSense SDK. To use, override [initCamera]
+/// and [disposeCamera], then override [sendFrames] to retreive and send images. Override
+/// [updateDetails] to be notified when the current [CameraDetails] have changed, but the common
+/// cases such as stopping and starting the camera will be handled for you.
+///
+/// You may use [sendStatus], [sendLog], or [sendFrame] to send data to the Dashboard. Do not try
+/// to communicate directly as only the parent isolate can access the network.
+///
+/// This class manages a few camera-independent details, such as:
+/// - periodically sending the camera's current status to the Dashboard
+/// - periodically logging how many frames were successfully read
+/// - periodically calling [sendFrames] to read the camera
+/// - calling [updateDetails] when a new [VideoCommand] arrives.
 abstract class CameraIsolate extends IsolateChild<IsolatePayload, VideoCommand> {
   /// Holds the current details of the camera.
   final CameraDetails details;
@@ -78,23 +92,28 @@ abstract class CameraIsolate extends IsolateChild<IsolatePayload, VideoCommand>
   /// This is separate from [dispose] so the isolate can keep reporting its status.
   void disposeCamera();
 
-  /// Reads frame/s from the camera and sends it/them.
+  /// Reads a frame from the camera and sends it to the dashboard.
+  ///
+  /// When overriding this function, be sure to check for errors, such as:
+  /// - If the camera does not respond, alert the dashboard
+  /// - If the frame is too large, reduces the quality (increases JPG compression)
+  /// - If the quality is already low, alert the dashboard
   Future<void> sendFrames();
 
   /// Sends an individual frame to the dashboard.
   ///
   /// This function also checks if the frame is too big to send, and if so,
   /// lowers the JPG quality by 1%. If the quality reaches 25% (visually noticeable),
   /// an error is logged instead.
-  void sendFrame(OpenCVImage image, {CameraDetails? detailsOverride}) {
+  void sendFrame(Uint8List image, {CameraDetails? detailsOverride}) {
     final details = detailsOverride ?? this.details;
-    if (image.data.length < maxPacketLength) {  // Frame can be sent
+    if (image.length < maxPacketLength) {  // Frame can be sent
       send(FramePayload(details: details, image: image));
     } else if (details.quality > 25) {  // Frame too large, lower quality
       sendLog(LogLevel.debug, "Lowering quality for $name from ${details.quality}");
       details.quality -= 1;  // maybe next frame can send
     } else {  // Frame too large, quality cannot be lowered
-      sendLog(LogLevel.warning, "Frame from camera $name are too large (${image.data.length})");
+      sendLog(LogLevel.warning, "Frame from camera $name are too large (${image.length})");
       updateDetails(CameraDetails(status: CameraStatus.FRAME_TOO_LARGE));
     }
   }

lib/src/isolates/opencv.dart

@@ -1,25 +1,20 @@
-import "dart:ffi";
-
-import "package:opencv_ffi/opencv_ffi.dart";
+import "package:opencv_dart/opencv_dart.dart";
 import "package:burt_network/burt_network.dart";
 
-import "package:video/video.dart";
+import "package:video/utils.dart";
+import "child.dart";
 
-/// An isolate that is spawned to manage one camera.
-///
-/// This class accepts [VideoCommand]s and calls [updateDetails] with the newly-received details.
-/// When a frame is read, instead of sending the [VideoData], this class sends only the pointer
-/// to the [OpenCVImage] via the [IsolatePayload] class, and the image is read by the parent isolate.
+/// A [CameraIsolate] that reads cameras using `package:opencv_dart`.
 class OpenCVCameraIsolate extends CameraIsolate {
   /// The native camera object from OpenCV.
-  late Camera camera;
+  late final VideoCapture camera;
   /// Creates a new manager for the given camera and default details.
   OpenCVCameraIsolate({required super.details});
 
   @override
   void initCamera() {
     camera = getCamera(name);
-    camera.setResolution(details.resolutionWidth, details.resolutionHeight);
+    camera.setResolution(width: details.resolutionWidth, height: details.resolutionHeight);
     if (!camera.isOpened) {
       sendLog(LogLevel.warning, "Camera $name is not connected");
       updateDetails(CameraDetails(status: CameraStatus.CAMERA_DISCONNECTED));
@@ -30,34 +25,30 @@ class OpenCVCameraIsolate extends CameraIsolate {
   void disposeCamera() => camera.dispose();
 
   @override
-  void updateDetails(CameraDetails newDetails, {bool restart = false}) {
-    camera.setResolution(details.resolutionWidth, details.resolutionHeight);
+  void updateDetails(CameraDetails newDetails) {
     super.updateDetails(newDetails);
+    camera.setResolution(width: details.resolutionWidth, height: details.resolutionHeight);
     camera.zoom = details.zoom;
     camera.pan = details.pan;
     camera.tilt = details.tilt;
     camera.focus = details.focus;
     camera.autofocus = details.focus;
   }
 
-  /// Reads a frame from the camera and sends it to the dashboard.
-  ///
-  /// Checks for multiple errors along the way:
-  /// - If the camera does not respond, alerts the dashboard
-  /// - If the frame is too large, reduces the quality (increases JPG compression)
-  /// - If the quality is already low, alerts the dashboard
   @override
   Future<void> sendFrames() async {
-    final matrix = camera.getFrame();
-    if (matrix == nullptr) return;
+    final (success, matrix) = camera.read();
+    if (!success) return;
     // detectAndAnnotateFrames(matrix);
-    final frame = encodeJpg(matrix, quality: details.quality);
+    final frame = matrix.encodeJpg(quality: details.quality);
     matrix.dispose();
+
     if (frame == null) {  // Error getting the frame
       sendLog(LogLevel.warning, "Camera $name didn't respond");
       updateDetails(CameraDetails(status: CameraStatus.CAMERA_NOT_RESPONDING));
       return;
     }
+
     sendFrame(frame);
     fpsCount++;
   }

lib/src/isolates/parent.dart

@@ -1,7 +1,6 @@
 import "dart:async";
 import "dart:io";
 
-import "package:opencv_ffi/opencv_ffi.dart" as opencv;
 import "package:typed_isolate/typed_isolate.dart";
 import "package:burt_network/burt_network.dart";
 
@@ -14,7 +13,7 @@ final autonomySocket = SocketInfo(address: InternetAddress("192.168.1.30"), port
 ///
 /// With one isolate per camera, each camera can read in parallel. This class sends [VideoCommand]s
 /// from the dashboard to the appropriate [CameraIsolate], and receives [IsolatePayload]s which it uses
-/// to read an [opencv.OpenCVImage] from native memory and send to the dashboard. By not sending the frame
+/// to read an image from native memory and send to the dashboard. By not sending the frame
 /// from child isolate to the parent (just the pointer), we save a whole JPG image's worth of bytes
 /// from every camera, every frame, every second. That could be up to 5 MB per second of savings.
 class VideoController extends IsolateParent<VideoCommand, IsolatePayload>{
@@ -50,9 +49,7 @@ class VideoController extends IsolateParent<VideoCommand, IsolatePayload>{
       case DetailsPayload():
         collection.videoServer.sendMessage(VideoData(details: data.details));
       case FramePayload():
-        final frame = data.frame;
-        collection.videoServer.sendMessage(VideoData(frame: frame.data, details: data.details));
-        frame.dispose();
+        collection.videoServer.sendMessage(VideoData(frame: data.image, details: data.details));
       case DepthFramePayload():
         collection.videoServer.sendMessage(VideoData(frame: data.frame.depthFrame), destination: autonomySocket);
         data.dispose();

lib/src/isolates/payload.dart

@@ -1,51 +1,38 @@
 import "dart:ffi";
+import "dart:typed_data";
 
 import "package:burt_network/burt_network.dart";
-import "package:opencv_ffi/opencv_ffi.dart" show OpenCVImage;
-
 import "package:video/video.dart";
 
 /// A payload containing some data to report back to the parent isolate.
-/// 
+///
 /// Instead of having nullable fields on this class, we subclass it and provide
 /// only the relevant fields for each subclass. That way, for example, you cannot
 /// accidentally send a frame without a [CameraDetails].
 sealed class IsolatePayload { const IsolatePayload(); }
 
-/// A payload representing the status of the given camera. 
+/// A payload representing the status of the given camera.
 class DetailsPayload extends IsolatePayload {
   /// The details being sent.
   final CameraDetails details;
   /// A const constructor.
   const DetailsPayload(this.details);
 }
 
-/// A container for a pointer to a native buffer that can be sent across isolates. 
-/// 
+/// A container for a pointer to a native buffer that can be sent across isolates.
+///
 /// Sending a buffer across isolates would mean that data is copied, which is not ideal for
 /// buffers containing an entire JPG image, from multiple isolates, multiple frames per second.
 /// Since we cannot yet send FFI pointers across isolates, we have to send its raw address.
 class FramePayload extends IsolatePayload {
   /// The details of the camera this frame came from.
   final CameraDetails details;
-  /// The address in FFI memory this frame starts at.
-  final int address;
-  /// The length of this frame in bytes.
-  final int length;
-
-  /// A const constructor.
-  FramePayload({required this.details, required OpenCVImage image}) : 
-    address = image.pointer.address,
-    length = image.data.length;
 
-  /// The underlying data held at [address]. 
-  /// 
-  /// This cannot be a normal field as [Pointer]s cannot be sent across isolates, and this should
-  /// not be a getter because the underlying memory needs to be freed and cannot be used again. 
-  OpenCVImage get frame => OpenCVImage(pointer: Pointer.fromAddress(address), length: length);
+  /// The image to send.
+  Uint8List image;
 
-  /// Frees the data in this frame.
-  void dispose() => frame.dispose();
+  /// A const constructor.
+  FramePayload({required this.details, required this.image});
 }
 
 /// A class to send log messages across isolates. The parent isolate is responsible for logging.
@@ -63,11 +50,11 @@ class DepthFramePayload extends IsolatePayload {
   /// The address of the data in memory, since pointers cannot be sent across isolates.
   final int address;
   /// Saves the address of the pointer to send across isolates.
-  DepthFramePayload(Pointer<NativeFrames> pointer) : 
+  DepthFramePayload(Pointer<NativeFrames> pointer) :
     address = pointer.address;
 
   /// The native frame being referenced by this pointer.
-  Pointer<NativeFrames> get frame => Pointer<NativeFrames>.fromAddress(address); 
+  Pointer<NativeFrames> get frame => Pointer<NativeFrames>.fromAddress(address);
 
   /// Frees the memory associated with the frame.
   void dispose() => frame.dispose();