Merge remote-tracking branch 'upstream/main' into clear-data

.github/ISSUE_TEMPLATE/bug_form.yml

@@ -0,0 +1,58 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: describe-bug
+    attributes:
+      label: Describe the bug
+      description: What behaviour was expected and what actually happened?
+      placeholder: null
+    validations:
+      required: true
+  - type: textarea
+    id: steps
+    attributes:
+      label: To Reproduce
+      description: What steps would we take to reproduce the behaviour
+      placeholder: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....'
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be
+        automatically formatted into code, so no need for backticks.
+      render: shell
+  - type: textarea
+    id: image-upload
+    attributes:
+      label: Upload Screenshot
+      description: Please upload an image that helps illustrate the issue.
+    validations:
+      required: false
+  - type: dropdown
+    id: OS
+    attributes:
+      label: What Operating System did you use?
+      multiple: true
+      options:
+        - macOS
+        - Apple iOS
+        - Microsoft Windows
+        - Google Android
+        - Linux
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: other
+    attributes:
+      label: Additional Context
+      description: null

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+blank_issues_enabled: true
+version: 0.1

.github/ISSUE_TEMPLATE/new_feature.yml

@@ -0,0 +1,28 @@
+name: "Feature Request"
+description: Submit a proposal for a new OpenAdapt feature
+labels: enhancement
+body:
+  - type: textarea
+    id: feature-request
+    validations:
+      required: true
+    attributes:
+      label: Feature request
+      description: A clear and concise description of the feature proposal. Please provide links to any relevant resources.
+
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: Please outline the purpose for the proposal (e.g., is it related to a problem?). Add any relevant links (e.g. GotHub issues).
+      placeholder: I'm always frustrated when [...] so this feature would [...].
+
+  - type: textarea
+    id: other
+    validations:
+      required: false
+    attributes:
+      label: Other
+      description: Any additional details

.github/pull_request_template.md

@@ -0,0 +1,25 @@
+<!-- Thanks for submitting a pull request! Please provide  information so we can quickly review your pull request. -->
+
+**What kind of change does this PR introduce?**
+
+<!-- E.g. a bugfix, feature, refactoring, etc… -->
+
+**Checklist**
+* [ ] My code follows the style guidelines of this project
+* [ ] I have pefomed a self-review of my code
+* [ ] If applicable, I have added tests to prove my fix is functional/effective
+* [ ] I have linted my code locally prior to submission
+* [ ] I have commented my code, particularly in hard-to-understand areas
+* [ ] I have made corresponding changes to the documentation (e.g. README.md, requirements.txt)
+* [ ] New and existing unit tests pass locally with my changes
+
+**Summary**
+
+<!-- Explain the **motivation** for making this change. What existing problem does the pull request solve? Try to link to an open issue. -->
+
+**How can your code be run and tested?**
+
+<!-- See the README.md for examples. -->
+
+
+**Other information**

CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# How to contribute
+
+We would love to implement your contributions to this project! We simply ask that you observe the following guidelines.  
+
+## Code Style
+
+This project follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
+
+
+## Creating Issue
+In order to effectively communicate any bugs or request new features, please select the appropriate form. If none of the options suit your needs, you can click on "Open a blank issue" located at the bottom.
+
+## Testing
+[GitHub Actions](https://github.com/MLDSAI/OpenAdapt/actions/new) are automatically run on each pull request to ensure consistent behaviour and style. The Actions are composed of PyTest, [black](https://github.com/psf/black) and [flake8](https://flake8.pycqa.org/en/latest/user/index.html).
+
+You can run these tests on your own computer by downloading the depencencies in requirements.txt and then running pytest in the root directory. 
+
+## Pull Request Format
+
+To speed up the review process, please use the provided pull request template and create a draft pull request to get initial feedback. 
+
+The pull request template includes areas to explain the changes, and a checklist with boxes for code style, testing, and documenttation.
+
+## Submitting Changes
+
+1. Fork the current repository
+2. Make a branch to work on, or use the main branch
+3. Push desired changes onto the branch in step 2
+4. Submit a pull request with the branch in step 2 as the head ref and the MLDSAI/OpenAdapt main as the base ref
+     - Note: may need to click "compare across forks"
+5. Update pull request using feedback
+     - this step may not be necessary, or may need to be repeated
+6. Celebrate contributing to OpenAdapt!

install/install_openadapt.ps1

@@ -14,6 +14,7 @@ $VCRedistInstaller = "vc_redist.x64.exe"
 $VCRedistInstallerLoc = "https://aka.ms/vs/17/release/vc_redist.x64.exe"
 $VCRedistRegPath = "HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\X64"
 
+# TODO: Add Tesseract OCR installation: https://tesseract-ocr.github.io/tessdoc/Installation.html
 
 # Return true if a command/exe is available
 function CheckCMDExists() {

install/install_openadapt.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+# TODO: Add Tesseract OCR installation: https://tesseract-ocr.github.io/tessdoc/Installation.html
+
 # Run a command and ensure it did not fail
 RunAndCheck() {
     res=$($1)

openadapt/config.py

@@ -30,10 +30,58 @@
     "RECORD_READ_ACTIVE_ELEMENT_STATE": False,
     # TODO: remove?
     "REPLAY_STRIP_ELEMENT_STATE": True,
+    # IGNORES WARNINGS (PICKLING, ETC.)
+    "IGNORE_WARNINGS": False,
     # ACTION EVENT CONFIGURATIONS
     "ACTION_TEXT_SEP": "-",
     "ACTION_TEXT_NAME_PREFIX": "<",
-    "ACTION_TEXT_NAME_SUFFIX": ">"
+    "ACTION_TEXT_NAME_SUFFIX": ">",
+    # SCRUBBING CONFIGURATIONS
+    "SCRUB_ENABLED": True,
+    "SCRUB_CHAR": "*",
+    "SCRUB_LANGUAGE": "en",
+    # TODO support lists in getenv_fallback
+    "SCRUB_FILL_COLOR": (255, 0, 0),
+    "SCRUB_CONFIG_TRF": {
+        "nlp_engine_name": "spacy",
+        "models": [{"lang_code": "en", "model_name": "en_core_web_trf"}],
+    },
+    "SCRUB_IGNORE_ENTITIES": [
+        # 'US_PASSPORT',
+        # 'US_DRIVER_LICENSE',
+        # 'CRYPTO',
+        # 'UK_NHS',
+        # 'PERSON',
+        # 'CREDIT_CARD',
+        # 'US_BANK_NUMBER',
+        # 'PHONE_NUMBER',
+        # 'US_ITIN',
+        # 'AU_ABN',
+        "DATE_TIME",
+        # 'NRP',
+        # 'SG_NRIC_FIN',
+        # 'AU_ACN',
+        # 'IP_ADDRESS',
+        # 'EMAIL_ADDRESS',
+        "URL",
+        # 'IBAN_CODE',
+        # 'AU_TFN',
+        # 'LOCATION',
+        # 'AU_MEDICARE',
+        # 'US_SSN',
+        # 'MEDICAL_LICENSE'
+    ],
+    "SCRUB_KEYS_HTML": [
+        "text",
+        "canonical_text",
+        "title",
+        "state",
+        "task_description",
+        "key_char",
+        "canonical_key_char",
+        "key_vk",
+        "children",
+    ],
 }
 
 
@@ -62,53 +110,21 @@ def getenv_fallback(var_name):
             logger.info(f"{key}={val}")
 
 
-# SCRUBBING CONFIGURATIONS
-SCRUB_ENABLED = True
-SCRUB_CHAR = "*"
-SCRUB_LANGUAGE = "en"
-SCRUB_CONFIG_TRF = {
-    "nlp_engine_name": "spacy",
-    "models": [
-        {
-        "lang_code": "en", 
-        "model_name": "en_core_web_trf"
-        }
-    ],
-}
-DEFAULT_SCRUB_FILL_COLOR = (255,0,0)
-SCRUB_IGNORE_ENTITIES = [
-    # 'US_PASSPORT',
-    # 'US_DRIVER_LICENSE',
-    # 'CRYPTO',
-    # 'UK_NHS',
-    # 'PERSON',
-    # 'CREDIT_CARD',
-    # 'US_BANK_NUMBER',
-    # 'PHONE_NUMBER',
-    # 'US_ITIN',
-    # 'AU_ABN',
-    "DATE_TIME",
-    # 'NRP',
-    # 'SG_NRIC_FIN',
-    # 'AU_ACN',
-    # 'IP_ADDRESS',
-    # 'EMAIL_ADDRESS',
-    "URL",
-    # 'IBAN_CODE',
-    # 'AU_TFN',
-    # 'LOCATION',
-    # 'AU_MEDICARE',
-    # 'US_SSN',
-    # 'MEDICAL_LICENSE'
-]
-SCRUB_KEYS_HTML = [
-    "text",
-    "canonical_text",
-    "title",
-    "state",
-    "task_description",
-    "key_char",
-    "canonical_key_char",
-    "key_vk",
-    "children",
-]
+def filter_log_messages(data):
+    """
+    This function filters log messages by ignoring any message that contains a specific string.
+
+    Args:
+      data: The input parameter "data" is expected to be data from a loguru logger.
+
+    Returns:
+      a boolean value indicating whether the message in the input data should be ignored or not. If the
+    message contains any of the messages in the `messages_to_ignore` list, the function returns `False`
+    indicating that the message should be ignored. Otherwise, it returns `True` indicating that the
+    message should not be ignored.
+    """
+    # TODO: ultimately, we want to fix the underlying issues, but for now, we can ignore these messages
+    messages_to_ignore = [
+        "Cannot pickle Objective-C objects",
+    ]
+    return not any(msg in data["message"] for msg in messages_to_ignore)

openadapt/models.py

@@ -269,6 +269,18 @@ def take_screenshot(cls):
         sct_img = utils.take_screenshot()
         screenshot = Screenshot(sct_img=sct_img)
         return screenshot
+
+    def crop_active_window(self, action_event):
+        window_event = action_event.window_event
+        width_ratio, height_ratio = utils.get_scale_ratios(action_event)
+
+        x0 = window_event.left * width_ratio
+        y0 = window_event.top * height_ratio
+        x1 = x0 + window_event.width * width_ratio
+        y1 = y0 + window_event.height * height_ratio
+
+        box = (x0, y0, x1, y1)
+        self._image = self._image.crop(box)
 
 
 class WindowEvent(db.Base):