dora-rs integrated

.gitignore

@@ -1 +1,2 @@
-node_modules/
+node_modules/
+/Images

bot.py

@@ -1,46 +1,77 @@
+import sys
+
+sys.stdin.reconfigure(encoding='utf-8')
+sys.stdout.reconfigure(encoding='utf-8')
+
 from javascript import require, On
-mineflayer = require('mineflayer')
-pathfinder = require('mineflayer-pathfinder')
 from dora import Node
 import pyarrow as pa
+from typing import Callable, Optional, Union
+from dora import DoraStatus
+from selenium import webdriver
+
+mineflayer = require('mineflayer')
+viewer = require('prismarine-viewer').mineflayer
+pathfinder = require('mineflayer-pathfinder')
+
 RANGE_GOAL = 1
 BOT_USERNAME = 'python'
-
+PORT = 59661
 node = Node()
 
-
 bot = mineflayer.createBot({
-  'host': 'localhost',
-  'port': 11111, 
+  #'host': '3000',
+  'port': PORT, 
   'username': BOT_USERNAME
 })
 
-bot.loadPlugin(pathfinder.pathfinder)
-print("Started mineflayer")
+#bot.loadPlugin(pathfinder.pathfinder)
+#print("Started mineflayer")
+
+#options = Options()
+#options.add_argument("--headless")
 
-@On(bot, 'spawn')
-def handle(*args):
-  print("I spawned 👋")
+
+def main(driver):
   movements = pathfinder.Movements(bot)
+  for i in range(100):
+    dora_event = node.next()
+    if dora_event is None:
+      print("No event", flush=True)
+      return
+    if dora_event["type"] == "INPUT":
+      match dora_event["id"]:
+        case "tick":
+          driver.save_screenshot("../Images/screenshot.png")
+          node.send_output('image', pa.array([]))
+"""
+    @On(bot, 'chat')
+    def handleMsg(this, sender, message, *args):
+      
+      node.send_output('chat', pa.array([message]))
+      print("Got message", sender, message)
+      if sender and (sender != BOT_USERNAME):
+        bot.chat('Hi, you said ' + message)
+        if 'come' in message:
+          player = bot.players[sender]
+          print("Target", player)
+          target = player.entity
+          if not target:
+            bot.chat("I don't see you !")
+            return
+
+          pos = target.position
+          bot.pathfinder.setMovements(movements)
+          bot.pathfinder.setGoal(pathfinder.goals.GoalNear(pos.x, pos.y, pos.z, RANGE_GOAL))"""
 
-  @On(bot, 'chat')
-  def handleMsg(this, sender, message, *args):
-    #node.send_output('chat', pa.array([message]))
-    print("Got message", sender, message)
-    if sender and (sender != BOT_USERNAME):
-      bot.chat('Hi, you said ' + message)
-      if 'come' in message:
-        player = bot.players[sender]
-        print("Target", player)
-        target = player.entity
-        if not target:
-          bot.chat("I don't see you !")
-          return
-
-        pos = target.position
-        bot.pathfinder.setMovements(movements)
-        bot.pathfinder.setGoal(pathfinder.goals.GoalNear(pos.x, pos.y, pos.z, RANGE_GOAL))
-
-@On(bot, "end")
-def handle(*args):
+def end(*args):
   print("Bot ended!", args)
+
+if __name__ == "__main__":
+  viewer(bot, {'port': 3000, 'firstPerson': True})
+  options = webdriver.FirefoxOptions()
+  options.add_argument("--headless")
+  options.add_argument("--window-size=960,540")
+  driver = webdriver.Firefox(options=options)
+  driver.get("localhost:3000")
+  main(driver)

graphs/dataflow.yml

@@ -2,14 +2,14 @@ nodes:
   - id: bot
     custom:
       source: python
-      args: ../basic.py
+      args: ../bot.py
       inputs:
-        tick: dora/timer/millis/50000
+        tick: dora/timer/millis/500
       outputs:
         - image
   - id: plot
     custom:
       source: python
       args: ../plot.py
       inputs:
-        chat: bot/image
+        image: bot/image

object_detection.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from typing import Callable, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+import torch
+
+from dora import DoraStatus
+
+pa.array([])
+
+CAMERA_WIDTH = 960
+CAMERA_HEIGHT = 540
+
+
+class Operator:
+    """
+    Infering object from images
+    """
+
+    def __init__(self):
+        self.model = torch.hub.load("ultralytics/yolov5", "yolov5n")
+
+    def on_event(
+        self,
+        dora_event: dict,
+        send_output: Callable[[str, Union[bytes, pa.Array], Optional[dict]], None],
+    ) -> DoraStatus:
+        if dora_event["type"] == "INPUT":
+            return self.on_input(dora_event, send_output)
+        return DoraStatus.CONTINUE
+
+    def on_input(
+        self,
+        dora_input: dict,
+        send_output: Callable[[str, Union[bytes, pa.array], Optional[dict]], None],
+    ) -> DoraStatus:
+        """Handle image
+        Args:
+            dora_input (dict): Dict containing the "id", "value", and "metadata"
+            send_output Callable[[str, bytes | pa.Array, Optional[dict]], None]:
+                Function for sending output to the dataflow:
+                - First argument is the `output_id`
+                - Second argument is the data as either bytes or `pa.Array`
+                - Third argument is dora metadata dict
+                e.g.: `send_output("bbox", pa.array([100], type=pa.uint8()), dora_event["metadata"])`
+        """
+
+        frame = dora_input["value"].to_numpy().reshape((CAMERA_HEIGHT, CAMERA_WIDTH, 3))
+        frame = frame[:, :, ::-1]  # OpenCV image (BGR to RGB)
+        results = self.model(frame)  # includes NMS
+        arrays = pa.array(np.array(results.xyxy[0].cpu()).ravel())
+        send_output("bbox", arrays, dora_input["metadata"])
+        return DoraStatus.CONTINUE