fix

puterbot/config.py

@@ -17,6 +17,7 @@
     "OPENAI_MODEL_NAME": "gpt-3.5-turbo",
     # may incur significant performance penalty
     "RECORD_READ_ACTIVE_ELEMENT_STATE": False,
+    # TODO: remove?
     "REPLAY_STRIP_ELEMENT_STATE": True,
 }
 

puterbot/playback.py

@@ -57,7 +57,8 @@ def play_key_event(event, keyboard_controller, canonical=True):
 
 
 def play_action_event(event, mouse_controller, keyboard_controller):
-    if event.children:
+    # currently we use children to replay type events
+    if event.children and event.name in KEY_EVENTS:
         for child in event.children:
             play_action_event(child, mouse_controller, keyboard_controller)
     else:

puterbot/record.py

@@ -664,7 +664,7 @@ def record(
 
     plot_performance(recording_timestamp, perf_q)
 
-    logger.info("done")
+    logger.info(f"saved {recording_timestamp=}")
 
 
 if __name__ == "__main__":

puterbot/strategies/base.py

@@ -26,7 +26,6 @@ def __init__(
     ):
         self.recording = recording
         self.max_frame_times = max_frame_times
-        self.processed_action_events = recording.processed_action_events
         self.action_events = []
         self.screenshots = []
         self.window_events = []
@@ -53,9 +52,14 @@ def run(self):
                 )
             except StopIteration:
                 break
+            if self.action_events:
+                prev_action_event = self.action_events[-1]
+                assert prev_action_event.timestamp <= action_event.timestamp, (
+                    prev_action_event, action_event
+                )
             self.log_fps()
-            self.action_events.append(action_event)
             if action_event:
+                self.action_events.append(action_event)
                 action_event_dict = utils.rows2dicts(
                     [action_event],
                     drop_constant=False,
@@ -64,11 +68,15 @@ def run(self):
                     f"action_event=\n"
                     f"{pformat(action_event_dict)}"
                 )
-                playback.play_action_event(
-                    action_event,
-                    mouse_controller,
-                    keyboard_controller,
-                )
+                try:
+                    playback.play_action_event(
+                        action_event,
+                        mouse_controller,
+                        keyboard_controller,
+                    )
+                except Exception as exc:
+                    logger.exception(exc)
+                    import ipdb; ipdb.set_trace()
 
     def log_fps(self):
         t = time.time()

puterbot/strategies/naive.py

@@ -13,7 +13,7 @@
 
 
 DISPLAY_EVENTS = False
-PROCESS_EVENTS = False
+PROCESS_EVENTS = True
 REPLAY_EVENTS = True
 SLEEP = False
 
@@ -25,23 +25,28 @@ def __init__(
         recording: models.Recording,
         display_events=DISPLAY_EVENTS,
         replay_events=REPLAY_EVENTS,
+        process_events=PROCESS_EVENTS,
         sleep=SLEEP,
     ):
         super().__init__(recording)
         self.display_events = display_events
         self.replay_events = replay_events
+        self.process_events = process_events
         self.sleep = sleep
         self.prev_timestamp = None
         self.action_event_idx = -1
-        event_dicts = utils.rows2dicts(self.processed_action_events)
-        logger.info(f"event_dicts=\n{pformat(event_dicts)}")
+        #event_dicts = utils.rows2dicts(self.processed_action_events)
+        #logger.info(f"event_dicts=\n{pformat(event_dicts)}")
 
     def get_next_action_event(
         self,
         screenshot: models.Screenshot,
         window_event: models.WindowEvent,
     ):
-        action_events = self.processed_action_events if PROCESS_EVENTS else self.recording.action_events
+        if self.process_events:
+            action_events = self.recording.processed_action_events
+        else:
+            action_events = self.recording.action_events
         self.action_event_idx += 1
         num_action_events = len(action_events)
         if self.action_event_idx >= num_action_events: