Merge branch 'main' into clear-data

.gitignore

@@ -13,13 +13,17 @@ cache
 
 # db
 *.db
+*.db-journal
 
 # VSCode
 .VSCode
+.vsCode
 
 # Generated performance charts
 performance
 
 # Generated when adding editable dependencies in requirements.txt (-e)
 src
+
+# MacOS file
 .DS_Store

README.md

@@ -31,6 +31,7 @@ source .venv/bin/activate
 pip install wheel
 pip install -r requirements.txt
 pip install -e .
+python -m spacy download en_core_web_trf
 alembic upgrade head
 pytest
 ```

alembic/env.py

@@ -7,12 +7,12 @@
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
-config = context.config
+alembic_config = context.config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
+if alembic_config .config_file_name is not None:
+    fileConfig(alembic_config .config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
@@ -21,6 +21,7 @@
 from openadapt.config import DB_URL
 from openadapt.models import *
 from openadapt.db import Base
+
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
@@ -46,7 +47,7 @@ def run_migrations_offline() -> None:
     script output.
 
     """
-    #url = config.get_main_option("sqlalchemy.url")
+    # url = config.get_main_option("sqlalchemy.url")
     url = get_url()
     context.configure(
         url=url,
@@ -67,7 +68,7 @@ def run_migrations_online() -> None:
     and associate a connection with the context.
 
     """
-    configuration = config.get_section(config.config_ini_section)
+    configuration = alembic_config .get_section(alembic_config .config_ini_section)
     configuration["sqlalchemy.url"] = get_url()
     connectable = engine_from_config(
         configuration=configuration,

openadapt/config.py

@@ -1,3 +1,14 @@
+"""Script containing configurations for the openadapt application.
+
+Usage:
+
+    from openadapt import config
+    ...
+    config.<setting>
+    ...
+
+"""
+
 import multiprocessing
 import os
 import pathlib
@@ -19,6 +30,10 @@
     "RECORD_READ_ACTIVE_ELEMENT_STATE": False,
     # TODO: remove?
     "REPLAY_STRIP_ELEMENT_STATE": True,
+    # ACTION EVENT CONFIGURATIONS
+    "ACTION_TEXT_SEP": "-",
+    "ACTION_TEXT_NAME_PREFIX": "<",
+    "ACTION_TEXT_NAME_SUFFIX": ">"
 }
 
 
@@ -39,8 +54,61 @@ def getenv_fallback(var_name):
 DB_FNAME = getenv_fallback("DB_FNAME")
 DB_FPATH = ROOT_DIRPATH / DB_FNAME
 DB_URL = f"sqlite:///{DB_FPATH}"
+DIRNAME_PERFORMANCE_PLOTS = "performance"
 
 if multiprocessing.current_process().name == "MainProcess":
     for key, val in locals().items():
         if not key.startswith("_") and key.isupper():
             logger.info(f"{key}={val}")
+
+
+# SCRUBBING CONFIGURATIONS
+SCRUB_ENABLED = True
+SCRUB_CHAR = "*"
+SCRUB_LANGUAGE = "en"
+SCRUB_CONFIG_TRF = {
+    "nlp_engine_name": "spacy",
+    "models": [
+        {
+        "lang_code": "en", 
+        "model_name": "en_core_web_trf"
+        }
+    ],
+}
+DEFAULT_SCRUB_FILL_COLOR = (255,0,0)
+SCRUB_IGNORE_ENTITIES = [
+    # 'US_PASSPORT',
+    # 'US_DRIVER_LICENSE',
+    # 'CRYPTO',
+    # 'UK_NHS',
+    # 'PERSON',
+    # 'CREDIT_CARD',
+    # 'US_BANK_NUMBER',
+    # 'PHONE_NUMBER',
+    # 'US_ITIN',
+    # 'AU_ABN',
+    "DATE_TIME",
+    # 'NRP',
+    # 'SG_NRIC_FIN',
+    # 'AU_ACN',
+    # 'IP_ADDRESS',
+    # 'EMAIL_ADDRESS',
+    "URL",
+    # 'IBAN_CODE',
+    # 'AU_TFN',
+    # 'LOCATION',
+    # 'AU_MEDICARE',
+    # 'US_SSN',
+    # 'MEDICAL_LICENSE'
+]
+SCRUB_KEYS_HTML = [
+    "text",
+    "canonical_text",
+    "title",
+    "state",
+    "task_description",
+    "key_char",
+    "canonical_key_char",
+    "key_vk",
+    "children",
+]

openadapt/models.py

@@ -6,7 +6,7 @@
 import numpy as np
 import sqlalchemy as sa
 
-from openadapt import db, utils, window
+from openadapt import config, db, utils, window
 
 
 # https://groups.google.com/g/sqlalchemy/c/wlr7sShU6-k
@@ -133,9 +133,9 @@ def canonical_key(self):
         )
 
     def _text(self, canonical=False):
-        sep = self._text_sep
-        name_prefix = self._text_name_prefix
-        name_suffix = self._text_name_suffix
+        sep = config.ACTION_TEXT_SEP
+        name_prefix = config.ACTION_TEXT_NAME_PREFIX
+        name_suffix = config.ACTION_TEXT_NAME_SUFFIX
         if canonical:
             key_attr = self.canonical_key
             key_name_attr = self.canonical_key_name
@@ -201,10 +201,6 @@ def __str__(self):
         rval = " ".join(attrs)
         return rval
 
-    _text_sep = "-"
-    _text_name_prefix = "<"
-    _text_name_suffix = ">"
-
     @classmethod
     def from_children(cls, children_dicts):
         children = [

openadapt/record.py

@@ -22,7 +22,7 @@
 import fire
 import mss.tools
 
-from openadapt import config, crud, utils, window
+from openadapt import config, crud, scrub, utils, window
 
 
 EVENT_TYPES = ("screen", "action", "window")
@@ -389,7 +389,7 @@ def read_window_events(
             #   File "...\env\lib\site-packages\loguru\_logger.py", line 1964, in _log
             #       for handler in core.handlers.values):
             #   RuntimeError: dictionary changed size during iteration
-            _window_data = dict(window_data)
+            _window_data = window_data
             _window_data.pop("state")
             logger.info(f"{_window_data=}")
         if window_data != prev_window_data:
@@ -524,8 +524,7 @@ def record(
     """
 
     utils.configure_logging(logger, LOG_LEVEL)
-
-    logger.info(f"{task_description=}")
+    logger.info(f"{scrub.scrub_text(task_description)=}")
 
     recording = create_recording(task_description)
     recording_timestamp = recording.timestamp