Merge pull request #151 from MLDSAI/feat/window_state

Window state

alembic/versions/57d78d23087a_add_windowevent_state.py

@@ -0,0 +1,32 @@
+"""add WindowEvent.state
+
+Revision ID: 57d78d23087a
+Revises: 20f9c2afb42c
+Create Date: 2023-05-14 18:32:57.473479
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '57d78d23087a'
+down_revision = '20f9c2afb42c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('state', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_column('state')
+
+    # ### end Alembic commands ###

alembic/versions/b2dc41850120_window_window_id.py

@@ -0,0 +1,32 @@
+"""Window.window_id
+
+Revision ID: b2dc41850120
+Revises: d63569e4fb90
+Create Date: 2023-05-17 12:50:35.125610
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b2dc41850120'
+down_revision = 'd63569e4fb90'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('window_id', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_column('window_id')
+
+    # ### end Alembic commands ###

alembic/versions/d63569e4fb90_actionevent_element_state.py

@@ -0,0 +1,32 @@
+"""ActionEvent.element_state
+
+Revision ID: d63569e4fb90
+Revises: ec337f277666
+Create Date: 2023-05-16 21:43:00.120143
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd63569e4fb90'
+down_revision = 'ec337f277666'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('action_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('element_state', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('action_event', schema=None) as batch_op:
+        batch_op.drop_column('element_state')
+
+    # ### end Alembic commands ###

alembic/versions/ec337f277666_datetime_timestamp.py

@@ -0,0 +1,38 @@
+"""DateTime timestamp
+
+Revision ID: ec337f277666
+Revises: 57d78d23087a
+Create Date: 2023-05-16 17:51:00.061604
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec337f277666'
+down_revision = '57d78d23087a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('screenshot', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_screenshot_recording_timestamp_recording'), 'recording', ['recording_timestamp'], ['timestamp'])
+
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_window_event_recording_timestamp_recording'), 'recording', ['recording_timestamp'], ['timestamp'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_window_event_recording_timestamp_recording'), type_='foreignkey')
+
+    with op.batch_alter_table('screenshot', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_screenshot_recording_timestamp_recording'), type_='foreignkey')
+
+    # ### end Alembic commands ###

openadapt/cache.py

@@ -0,0 +1,38 @@
+from functools import wraps
+import time
+
+from joblib import Memory
+from loguru import logger
+
+
+from openadapt import config
+
+
+def default(val, default):
+    return val if val is not None else default
+
+
+def cache(dir_path=None, enabled=None, verbosity=None, **cache_kwargs):
+    """TODO"""
+
+    cache_dir_path = default(dir_path, config.CACHE_DIR_PATH)
+    cache_enabled = default(enabled, config.CACHE_ENABLED)
+    cache_verbosity = default(verbosity, config.CACHE_VERBOSITY)
+    def decorator(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            logger.debug(f"{cache_enabled=}")
+            if cache_enabled:
+                memory = Memory(cache_dir_path, verbose=cache_verbosity)
+                nonlocal fn
+                fn = memory.cache(fn, **cache_kwargs)
+                cache_hit = fn.check_call_in_cache(*args, **kwargs)
+                logger.debug(f"{fn=} {cache_hit=}")
+            start_time = time.time()
+            logger.debug(f"{fn=} {start_time=}")
+            rval = fn(*args, **kwargs)
+            duration = time.time() - start_time
+            logger.debug(f"{fn=} {duration=}")
+            return rval
+        return wrapper
+    return decorator

openadapt/common.py

@@ -14,3 +14,4 @@
     # processed
     "type",
 )
+ALL_EVENTS = tuple(list(MOUSE_EVENTS) + list(KEY_EVENTS))

openadapt/config.py

@@ -1,8 +1,45 @@
+import multiprocessing
+import os
 import pathlib
 
-ROOT_DIRPATH = pathlib.Path(__file__).parent.parent.resolve()
-DB_FNAME = "openadapt.db"
+from dotenv import load_dotenv
+from loguru import logger
+
+
+_DEFAULTS = {
+    "CACHE_DIR_PATH": ".cache",
+    "CACHE_ENABLED": True,
+    "CACHE_VERBOSITY": 0,
+    "DB_ECHO": False,
+    "DB_FNAME": "openadapt.db",
+    "OPENAI_API_KEY": None,
+    #"OPENAI_MODEL_NAME": "gpt-4",
+    "OPENAI_MODEL_NAME": "gpt-3.5-turbo",
+    # may incur significant performance penalty
+    "RECORD_READ_ACTIVE_ELEMENT_STATE": False,
+    # TODO: remove?
+    "REPLAY_STRIP_ELEMENT_STATE": True,
+}
+
+
+def getenv_fallback(var_name):
+    rval = os.getenv(var_name) or _DEFAULTS.get(var_name)
+    if rval is None:
+        raise ValueError(f"{var_name=} not defined")
+    return rval
 
+
+load_dotenv()
+
+for key in _DEFAULTS:
+    val = getenv_fallback(key)
+    locals()[key] = val
+
+ROOT_DIRPATH = pathlib.Path(__file__).parent.parent.resolve()
 DB_FPATH = ROOT_DIRPATH / DB_FNAME
 DB_URL = f"sqlite:///{DB_FPATH}"
-DB_ECHO = False
+
+if multiprocessing.current_process().name == "MainProcess":
+    for key, val in locals().items():
+        if not key.startswith("_") and key.isupper():
+            logger.info(f"{key}={val}")

openadapt/crud.py

@@ -87,6 +87,15 @@ def get_latest_recording():
     )
 
 
+def get_recording(timestamp):
+    return (
+        db
+        .query(Recording)
+        .filter(Recording.timestamp == timestamp)
+        .first()
+    )
+
+
 def _get(table, recording_timestamp):
     return (
         db
@@ -101,7 +110,7 @@ def get_action_events(recording):
     return _get(ActionEvent, recording.timestamp)
 
 
-def get_screenshots(recording, precompute_diffs=True):
+def get_screenshots(recording, precompute_diffs=False):
     screenshots = _get(Screenshot, recording.timestamp)
 
     for prev, cur in zip(screenshots, screenshots[1:]):

openadapt/events.py

@@ -5,31 +5,23 @@
 from scipy.spatial import distance
 import numpy as np
 
-from openadapt.common import KEY_EVENTS, MOUSE_EVENTS
-from openadapt.crud import (
-    get_action_events,
-    get_window_events,
-    get_screenshots,
-)
-from openadapt.models import ActionEvent
-from openadapt.utils import (
-    get_double_click_distance_pixels,
-    get_double_click_interval_seconds,
-    get_scale_ratios,
-    rows2dicts,
-)
+from openadapt import common, config, crud, models, utils
 
 
 MAX_PROCESS_ITERS = 1
 
 
-def get_events(recording, process=True, meta=None):
+def get_events(
+    recording,
+    process=True,
+    meta=None,
+):
     start_time = time.time()
-    action_events = get_action_events(recording)
-    window_events = get_window_events(recording)
-    screenshots = get_screenshots(recording)
+    action_events = crud.get_action_events(recording)
+    window_events = crud.get_window_events(recording)
+    screenshots = crud.get_screenshots(recording)
 
-    raw_action_event_dicts = rows2dicts(action_events)
+    raw_action_event_dicts = utils.rows2dicts(action_events)
     logger.debug(f"raw_action_event_dicts=\n{pformat(raw_action_event_dicts)}")
 
     num_action_events = len(action_events)
@@ -108,10 +100,12 @@ def make_parent_event(child, extra=None):
     extra = extra or {}
     for key, val in extra.items():
         event_dict[key] = val
-    return ActionEvent(**event_dict)
+    return models.ActionEvent(**event_dict)
 
 
-def merge_consecutive_mouse_move_events(events, by_diff_distance=True):
+# Set by_diff_distance=True to compute distance from mouse to screenshot diff
+# (computationally expensive but keeps more useful events)
+def merge_consecutive_mouse_move_events(events, by_diff_distance=False):
     """Merge consecutive mouse move events into a single move event"""
 
     _all_slowdowns = []
@@ -138,7 +132,7 @@ def get_merged_events(
         # (inclusive, exclusive)
         group_idx_tups = [(0, N)]
         if by_diff_distance:
-            width_ratio, height_ratio = get_scale_ratios(to_merge[0])
+            width_ratio, height_ratio = utils.get_scale_ratios(to_merge[0])
             close_idxs = []
             # TODO: improve performance, e.g. vectorization, resizing
             _all_dts = []
@@ -283,13 +277,13 @@ def get_timestamp_mappings(to_merge):
         double_click_distance = get_recording_attr(
             to_merge[0],
             "double_click_distance_pixels",
-            get_double_click_distance_pixels,
+            utils.get_double_click_distance_pixels,
         )
         logger.info(f"{double_click_distance=}")
         double_click_interval = get_recording_attr(
             to_merge[0],
             "double_click_interval_seconds",
-            get_double_click_interval_seconds,
+            utils.get_double_click_interval_seconds,
         )
         logger.info(f"{double_click_interval=}")
         press_to_press_t = {}
@@ -477,7 +471,7 @@ def remove_redundant_mouse_move_events(events):
 
 
     def is_target_event(event, state):
-        return event.name in ("click", "move")
+        return event.name in ("move", "click")
 
 
     def is_same_pos(e0, e1):
@@ -553,7 +547,7 @@ def include_merged_events(to_merge):
 
 
     for event in events:
-        assert event.name in MOUSE_EVENTS + KEY_EVENTS, event
+        assert event.name in common.ALL_EVENTS, event
         if is_target_event(event, state):
             to_merge.append(event)
         else:
@@ -594,6 +588,11 @@ def discard_unused_events(
 
 
 def process_events(action_events, window_events, screenshots):
+    # for debugging
+    _action_events = action_events
+    _window_events = window_events
+    _screenshots = screenshots
+
     num_action_events = len(action_events)
     num_window_events = len(window_events)
     num_screenshots = len(screenshots)