wip

alembic/versions/ec337f277666_datetime_timestamp.py

@@ -0,0 +1,38 @@
+"""DateTime timestamp
+
+Revision ID: ec337f277666
+Revises: 57d78d23087a
+Create Date: 2023-05-16 17:51:00.061604
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec337f277666'
+down_revision = '57d78d23087a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('screenshot', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_screenshot_recording_timestamp_recording'), 'recording', ['recording_timestamp'], ['timestamp'])
+
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_window_event_recording_timestamp_recording'), 'recording', ['recording_timestamp'], ['timestamp'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_window_event_recording_timestamp_recording'), type_='foreignkey')
+
+    with op.batch_alter_table('screenshot', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_screenshot_recording_timestamp_recording'), type_='foreignkey')
+
+    # ### end Alembic commands ###

puterbot/config.py

@@ -1,12 +1,16 @@
+import os
+import pathlib
+
 from dotenv import load_dotenv
 from loguru import logger
-import pathlib
 
 
 _DEFAULTS = {
-    "OPENAI_API_KEY": None,
-    "DB_FNAME": "openadapt.db",
     "DB_ECHO": False,
+    "DB_FNAME": "openadapt.db",
+    "OPENAI_API_KEY": None,
+    "OPENAI_SYSTEM_MESSAGE": "TODO",
+    "OPENAI_MODEL_NAME": "gpt-4",
 }
 
 

puterbot/crud.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 from loguru import logger
 import sqlalchemy as sa
 
@@ -87,6 +89,15 @@ def get_latest_recording():
     )
 
 
+def get_recording(timestamp):
+    return (
+        db
+        .query(Recording)
+        .filter(Recording.timestamp == timestamp)
+        .first()
+    )
+
+
 def _get(table, recording_timestamp):
     return (
         db

puterbot/events.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 import time
 
 from loguru import logger
@@ -83,7 +84,7 @@ def get_events(recording, process=True, meta=None):
 
         duration = action_events[-1].timestamp - action_events[0].timestamp
         if len(action_events) > 1:
-            assert duration > 0, duration
+            assert duration > timedelta(), duration
         meta["duration"] = format_num(duration, duration_raw)
 
     end_time = time.time()
@@ -304,7 +305,7 @@ def get_timestamp_mappings(to_merge):
                     dx = abs(event.mouse_x - prev_pressed_event.mouse_x)
                     dy = abs(event.mouse_y - prev_pressed_event.mouse_y)
                     if (
-                        dt <= double_click_interval and
+                        dt <= timedelta(seconds=double_click_interval) and
                         dx <= double_click_distance and
                         dy <= double_click_distance
                     ):
@@ -479,7 +480,7 @@ def remove_redundant_mouse_move_events(events):
 
 
     def is_target_event(event, state):
-        return event.name in ("click", "move")
+        return event.name == "move"
 
 
     def is_same_pos(e0, e1):
@@ -543,7 +544,7 @@ def merge_consecutive_action_events(
     """Merge consecutive action events into a single event"""
 
     num_events_before = len(events)
-    state = {"dt": 0}
+    state = {"dt": timedelta()}
     rval = []
     to_merge = []
 
@@ -596,6 +597,11 @@ def discard_unused_events(
 
 
 def process_events(action_events, window_events, screenshots):
+    # for debugging
+    _action_events = action_events
+    _window_events = window_events
+    _screenshots = screenshots
+
     num_action_events = len(action_events)
     num_window_events = len(window_events)
     num_screenshots = len(screenshots)

puterbot/models.py

@@ -14,23 +14,27 @@ class Recording(Base):
     __tablename__ = "recording"
 
     id = sa.Column(sa.Integer, primary_key=True)
-    timestamp = sa.Column(sa.Integer)
+    timestamp = sa.Column(sa.DateTime)
     monitor_width = sa.Column(sa.Integer)
     monitor_height = sa.Column(sa.Integer)
     double_click_interval_seconds = sa.Column(sa.Numeric(asdecimal=False))
     double_click_distance_pixels = sa.Column(sa.Numeric(asdecimal=False))
     platform = sa.Column(sa.String)
     task_description = sa.Column(sa.String)
 
-    action_events = sa.orm.relationship("ActionEvent", back_populates="recording")
+    action_events = sa.orm.relationship(
+        "ActionEvent",
+        back_populates="recording",
+        order_by="ActionEvent.timestamp",
+    )
 
 
 class ActionEvent(Base):
     __tablename__ = "action_event"
 
     id = sa.Column(sa.Integer, primary_key=True)
     name = sa.Column(sa.String)
-    timestamp = sa.Column(sa.Integer)
+    timestamp = sa.Column(sa.DateTime)
     recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp"))
     screenshot_timestamp = sa.Column(sa.ForeignKey("screenshot.timestamp"))
     window_event_timestamp = sa.Column(sa.ForeignKey("window_event.timestamp"))
@@ -69,7 +73,7 @@ def _key(self, key_name, key_char, key_vk):
 
     @property
     def key(self):
-        logger.debug(
+        logger.trace(
             f"{self.name=} {self.key_name=} {self.key_char=} {self.key_vk=}"
         )
         return self._key(
@@ -80,7 +84,7 @@ def key(self):
 
     @property
     def canonical_key(self):
-        logger.debug(
+        logger.trace(
             f"{self.name=} "
             f"{self.canonical_key_name=} "
             f"{self.canonical_key_char=} "
@@ -158,8 +162,8 @@ class Screenshot(Base):
     __tablename__ = "screenshot"
 
     id = sa.Column(sa.Integer, primary_key=True)
-    recording_timestamp = sa.Column(sa.Integer)
-    timestamp = sa.Column(sa.Integer)
+    recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp"))
+    timestamp = sa.Column(sa.DateTime)
     png_data = sa.Column(sa.LargeBinary)
     # TODO: replace prev with prev_timestamp?
 
@@ -216,8 +220,8 @@ class WindowEvent(Base):
     __tablename__ = "window_event"
 
     id = sa.Column(sa.Integer, primary_key=True)
-    recording_timestamp = sa.Column(sa.Integer)
-    timestamp = sa.Column(sa.Integer)
+    recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp"))
+    timestamp = sa.Column(sa.DateTime)
     state = sa.Column(sa.JSON)
     title = sa.Column(sa.String)
     left = sa.Column(sa.Integer)