add puterbot/, .gitignore

.gitignore

@@ -0,0 +1,12 @@
+# Envirionment
+.env
+
+# Python
+__pycache__
+cache
+*.egg-info
+.venv
+*~
+
+# Vim
+*.sw[m-p]

puterbot/README.md

@@ -0,0 +1,126 @@
+## Setup
+
+```
+git clone https://github.com/MLDSAI/puterbot.git
+cd puterbot
+python3.9 -m venv env
+source env/bin/activate
+pip install wheel
+pip install -r requirements
+```
+
+## Developing
+
+```
+pip install -e .
+```
+
+### Create database (if not exists) and Migrate to head
+
+```
+alembic upgrade head
+```
+
+### Generate migration (after editing a model)
+
+```
+alembic revision --autogenerate -m "<msg>"
+```
+
+### Run tests
+```
+pytest
+```
+
+## Running
+
+Record:
+```
+python puterbot/record.py
+```
+
+Visualize:
+```
+python puterbot/visualize.py
+```
+
+## Building (Windows only)
+
+install ms c++ build tools
+https://visualstudio.microsoft.com/visual-cpp-build-tools/
+
+install python 3.9.6
+https://www.python.org/ftp/python/3.9.6/python-3.9.6-amd64.exe
+
+trust pip hosts on proxy restricted windows machine:
+```
+pip config set global.trusted-host "pypi.org files.pythonhosted.org pypi.python.org" --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org
+```
+
+install tesseract
+```
+https://github.om/UB-Mannheim/tesseract/wiki/
+```
+
+add to path in the current shell
+```
+set PATH=%PATH%;%USERPROFILE%\AppData\Local\Tesseract-OCR
+```
+
+do it permanently
+```
+setx PATH "%PATH%;%USERPROFILE%\AppData\Local\Tesseract-OCR"
+```
+
+build
+```
+pyinstaller --noconfirm --add-data "resources\*.png;resources" audit.py
+```
+
+## Script dependencies
+
+install easyocr
+```
+pip install --user easyocr -i https://pypi.python.org/simple/
+```
+
+install kerasocr
+```
+pip install --user keras-ocr -i https://pypi.python.org/simple
+python -m pip install --user tensorflow -i https://pypi.python.org/simple --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org
+```
+
+install paddleocr
+```
+python -m pip install --user paddlepaddle
+````
+
+## Troubleshooting
+
+Apple Silicon:
+
+```
+$ python puterbot/record.py
+...
+This process is not trusted! Input event monitoring will not be possible until it is added to accessibility clients.
+```
+
+Solution:
+https://stackoverflow.com/a/69673312
+
+```
+Settings -> Security & Privacy
+Click on the Privacy tab
+Scroll and click on the Accessibility Row
+Click the +
+Navigate to /System/Applications/Utilities/ or wherever the Terminal.app is installed
+Click okay.
+```
+
+## Submitting an Issue
+
+Please submit any issues to https://github.com/MLDSAI/puterbot/issues with the
+following information:
+
+- Problem description (include any relevant console output and/or screenshots)
+- Steps to reproduce (required in order for others to help you)

puterbot/common.py

@@ -0,0 +1,16 @@
+MOUSE_EVENTS = (
+    # raw
+    "move",
+    "click",
+    "scroll",
+    # processed
+    "doubleclick",
+    "singleclick",
+)
+KEY_EVENTS = (
+    # raw
+    "press",
+    "release",
+    # processed
+    "type",
+)

puterbot/config.py

@@ -0,0 +1,8 @@
+import pathlib
+
+ROOT_DIRPATH = pathlib.Path(__file__).parent.parent.resolve()
+DB_FNAME = "puterbot.db"
+
+DB_FPATH = ROOT_DIRPATH / DB_FNAME
+DB_URL = f"sqlite:///{DB_FPATH}"
+DB_ECHO = False

puterbot/crud.py

@@ -0,0 +1,120 @@
+from loguru import logger
+import sqlalchemy as sa
+
+from puterbot.db import Session
+from puterbot.models import InputEvent, Screenshot, Recording, WindowEvent
+
+
+BATCH_SIZE = 1
+
+db = Session()
+input_events = []
+screenshots = []
+window_events = []
+
+
+def _insert(event_data, table, buffer=None):
+    """Insert using Core API for improved performance (no rows are returned)"""
+
+    db_obj = {
+        column.name: None
+        for column in table.__table__.columns
+    }
+    for key in db_obj:
+        if key in event_data:
+            val = event_data[key]
+            db_obj[key] = val
+            del event_data[key]
+
+    # make sure all event data was saved
+    assert not event_data, event_data
+
+    if buffer is not None:
+        buffer.append(db_obj)
+
+    if buffer is None or len(buffer) >= BATCH_SIZE:
+        to_insert = buffer or [db_obj]
+        result = db.execute(sa.insert(table), to_insert)
+        db.commit()
+        if buffer:
+            buffer.clear()
+        # Note: this does not contain the inserted row(s)
+        return result
+
+
+def insert_input_event(recording_timestamp, event_timestamp, event_data):
+    event_data = {
+        **event_data,
+        "timestamp": event_timestamp,
+        "recording_timestamp": recording_timestamp,
+    }
+    _insert(event_data, InputEvent, input_events)
+
+
+def insert_screenshot(recording_timestamp, event_timestamp, event_data):
+    event_data = {
+        **event_data,
+        "timestamp": event_timestamp,
+        "recording_timestamp": recording_timestamp,
+    }
+    _insert(event_data, Screenshot, screenshots)
+
+
+def insert_window_event(recording_timestamp, event_timestamp, event_data):
+    event_data = {
+        **event_data,
+        "timestamp": event_timestamp,
+        "recording_timestamp": recording_timestamp,
+    }
+    _insert(event_data, WindowEvent, window_events)
+
+
+def insert_recording(recording_data):
+    db_obj = Recording(**recording_data)
+    db.add(db_obj)
+    db.commit()
+    db.refresh(db_obj)
+    return db_obj
+
+
+def get_latest_recording():
+    return (
+        db
+        .query(Recording)
+        .order_by(sa.desc(Recording.timestamp))
+        .limit(1)
+        .first()
+    )
+
+
+def _get(table, recording_timestamp):
+    return (
+        db
+        .query(table)
+        .filter(table.recording_timestamp == recording_timestamp)
+        .order_by(table.timestamp)
+        .all()
+    )
+
+
+def get_input_events(recording):
+    return _get(InputEvent, recording.timestamp)
+
+
+def get_screenshots(recording, precompute_diffs=True):
+    screenshots = _get(Screenshot, recording.timestamp)
+
+    for prev, cur in zip(screenshots, screenshots[1:]):
+        cur.prev = prev
+    screenshots[0].prev = screenshots[0]
+
+    # TODO: store diffs
+    if precompute_diffs:
+        logger.info(f"precomputing diffs...")
+        [(screenshot.diff, screenshot.diff_mask) for screenshot in screenshots]
+
+    return screenshots
+
+
+def get_window_events(recording):
+    return _get(WindowEvent, recording.timestamp)

puterbot/db.py

@@ -0,0 +1,53 @@
+import sqlalchemy as sa
+from dictalchemy import DictableModel
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.schema import MetaData
+from sqlalchemy.ext.declarative import declarative_base
+
+from puterbot.config import DB_ECHO, DB_URL
+from puterbot.utils import EMPTY, row2dict
+
+
+NAMING_CONVENTION = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}
+
+
+class BaseModel(DictableModel):
+
+    __abstract__ = True
+
+    def __repr__(self):
+        params = ", ".join(
+            f"{k}={v!r}"  # !r converts value to string using repr (adds quotes)
+            for k, v in row2dict(self, follow=False).items()
+            if v not in EMPTY
+        )
+        return f"{self.__class__.__name__}({params})"
+
+
+def get_engine():
+    engine = sa.create_engine(
+        DB_URL,
+        echo=DB_ECHO,
+    )
+    return engine
+
+
+def get_base(engine):
+    metadata = MetaData(naming_convention=NAMING_CONVENTION)
+    Base = declarative_base(
+        cls=BaseModel,
+        bind=engine,
+        metadata=metadata,
+    )
+    return Base
+
+
+engine = get_engine()
+Base = get_base(engine)
+Session = sessionmaker(bind=engine)