Merge remote-tracking branch 'upstream/main' into share-magic-wormhole

.github/ISSUE_TEMPLATE/bug_form.yml

@@ -12,15 +12,15 @@ body:
     attributes:
       label: Describe the bug
       description: What behaviour was expected and what actually happened? Include screenshots and log output where useful.
-      value: I expected ... but what actually happened was ...
+      placeholder: I expected ... but what actually happened was ...
     validations:
       required: true
   - type: textarea
     id: steps
     attributes:
       label: To Reproduce
       description: What Operating System did you use and steps would we take to reproduce the behaviour?
-      value: "I use [macOS/Microsoft Windows].
+      placeholder: "I use [macOS/Microsoft Windows].
         Steps: 
         1. Go to '...' 
         2. Click on '....' 

.github/ISSUE_TEMPLATE/new_feature.yml

@@ -17,4 +17,4 @@ body:
     attributes:
       label: Motivation
       description: Please outline the purpose for the proposal (e.g., is it related to a problem?). Add any relevant links (e.g. GitHub issues).
-      value: I'm always frustrated when [...] so this feature would [...].
+      placeholder: I'm always frustrated when [...] so this feature would [...].

.github/pull_request_template.md

@@ -1,25 +1,26 @@
-<!-- Thanks for submitting a pull request! Please provide  information so we can quickly review your pull request. -->
+<!-- Thank you for submitting a pull request! To ensure a prompt review of your changes, please provide the following information. -->
 
 **What kind of change does this PR introduce?**
 
 <!-- E.g. a bugfix, feature, refactoring, etc… -->
 
+**Summary**
+
+<!-- Explain the **motivation** for making this change. What existing problem does the pull request solve? Try to link to an open issue. -->
+
 **Checklist**
-* [ ] My code follows the style guidelines of this project
-* [ ] I have pefomed a self-review of my code
+* [ ] My code follows the style guidelines of OpenAdapt
+* [ ] I have perfomed a self-review of my code
 * [ ] If applicable, I have added tests to prove my fix is functional/effective
 * [ ] I have linted my code locally prior to submission
 * [ ] I have commented my code, particularly in hard-to-understand areas
 * [ ] I have made corresponding changes to the documentation (e.g. README.md, requirements.txt)
 * [ ] New and existing unit tests pass locally with my changes
 
-**Summary**
-
-<!-- Explain the **motivation** for making this change. What existing problem does the pull request solve? Try to link to an open issue. -->
-
 **How can your code be run and tested?**
 
-<!-- See the README.md for examples. -->
+<!-- See the README.md for examples. Include test output. -->
 
 
-**Other information**
+**Other information**
+<!-- Delete this subheading if no additional context is needed. -->

.github/workflows/main.yml

@@ -0,0 +1,42 @@
+name: Python CI
+
+on:
+  push:
+    branches:
+      - '**'
+
+jobs:
+  run-ci:
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          pip install wheel
+          pip install -r requirements.txt
+          pip install -e .
+
+      - name: Check formatting with Black
+        run: |
+          black --check --exclude "(src|alembic)/" .
+
+      - name: Run headless tests
+        uses: coactions/setup-xvfb@v1
+        with:
+          run: python -m pytest
+          working-directory: ./  # Optional: Specify the working directory if needed
+          options:  # Optional: Add any additional options or arguments for pytest
+
+      - name: flake8 Lint
+        uses: py-actions/flake8@v2
+        with:
+          plugins: "flake8-docstrings"
+          extra-args: "--docstring-convention=google --exclude=alembic/versions"

.gitignore

@@ -19,8 +19,14 @@ cache
 .VSCode
 .vsCode
 
+#Idea
+.idea
+
 # Generated performance charts
 performance
 
 # Generated when adding editable dependencies in requirements.txt (-e)
 src
+
+# MacOS file
+.DS_Store

README.md

@@ -2,9 +2,16 @@
 
 # OpenAdapt: AI-First Process Automation with Transformers
 
-<img width="1512" alt="image" src="https://github.com/MLDSAI/OpenAdapt/assets/774615/60a85160-7b07-41b0-9295-2d1bfa6a0994">
+### Enormous volumes of mental labor are wasted on repetitive GUI workflows.
 
-([Slides](https://docs.google.com/presentation/d/e/2PACX-1vStYEzETEMtzWDGCizJt4L2rdQpoAQin8T2cZkLw6WqNBUtmBhaNm-1BrhvGOVNyFV9UPjyVfa0l6IC/pub?start=false&loop=false&delayms=3000))
+### Foundation Models (e.g. [GPT-4](https://openai.com/research/gpt-4), [ACT-1](https://www.adept.ai/blog/act-1)) are powerful automation tools.
+
+### OpenAdapt connects Foundation Models to GUIs:
+
+<img width="1473" alt="image" src="https://github.com/MLDSAI/OpenAdapt/assets/774615/5a760e4a-c596-4604-b1a4-a9563dce0fe7">
+
+
+([Slides](https://t.ly/7RGr))
 
 Welcome to OpenAdapt! This Python library implements AI-First Process Automation
 with the power of Transformers by:
@@ -27,6 +34,18 @@ The direction is adjacent to [Adept.ai](https://adept.ai/), with some key differ
 
 ## Install
 
+Recommended: Install with [Poetry](https://python-poetry.org/):
+```
+git clone https://github.com/MLDSAI/OpenAdapt.git
+cd OpenAdapt
+pip install poetry
+poetry install
+poetry shell
+alembic upgrade head
+pytest
+```
+
+Manual:
 ```
 git clone https://github.com/MLDSAI/OpenAdapt.git
 cd OpenAdapt

openadapt/app/__init__.py

@@ -0,0 +1,3 @@
+from openadapt.app.main import run_app
+
+run_app()

openadapt/app/build.py

@@ -0,0 +1,30 @@
+import os
+import subprocess
+from pathlib import Path
+
+import nicegui
+
+spec = [
+    "pyi-makespec",
+    f"{Path(__file__).parent}/main.py",
+    f"--icon={Path(__file__).parent}/assets/logo.ico",
+    "--name",
+    "OpenAdapt",  # name
+    # "--onefile", # trade startup speed for smaller file size
+    "--onedir",
+    "--windowed",  # prevent console appearing, only use with ui.run(native=True, ...)
+    "--add-data",
+    f"{Path(nicegui.__file__).parent}{os.pathsep}nicegui",
+]
+
+subprocess.call(spec)
+
+# add import sys ; sys.setrecursionlimit(sys.getrecursionlimit() * 5) to line 2 of OpenAdapt.spec
+with open("OpenAdapt.spec", "r+") as f:
+    lines = f.readlines()
+    lines[1] = "import sys ; sys.setrecursionlimit(sys.getrecursionlimit() * 5)\n"
+    f.seek(0)
+    f.truncate()
+    f.writelines(lines)
+
+subprocess.call(["pyinstaller", "OpenAdapt.spec"])

openadapt/app/cards.py

@@ -0,0 +1,88 @@
+import signal
+from nicegui import ui
+from subprocess import Popen
+from openadapt.app.objects.local_file_picker import LocalFilePicker
+from openadapt.app.util import set_dark, sync_switch
+
+PROC = None
+
+
+def settings(dark_mode):
+    with ui.dialog() as settings, ui.card():
+        s = ui.switch("Dark mode", on_change=lambda: set_dark(dark_mode, s.value))
+        sync_switch(s, dark_mode)
+        ui.button("Close", on_click=lambda: settings.close())
+
+    settings.open()
+
+
+def select_import(f):
+    async def pick_file():
+        result = await LocalFilePicker(".")
+        ui.notify(f"Selected {result[0]}" if result else "No file selected.")
+        selected_file.text = result[0] if result else ""
+        import_button.enabled = True if result else False
+
+    with ui.dialog() as import_dialog, ui.card():
+        with ui.column():
+            ui.button("Select File", on_click=pick_file).props("icon=folder")
+            selected_file = ui.label("")
+            selected_file.visible = False
+            import_button = ui.button(
+                "Import", on_click=lambda: f(selected_file.text, delete.value)
+            )
+            import_button.enabled = False
+            delete = ui.checkbox("Delete file after import")
+
+    import_dialog.open()
+
+
+def recording_prompt(options, record_button):
+    if PROC is None:
+        with ui.dialog() as dialog, ui.card():
+            ui.label("Enter a name for the recording: ")
+            ui.input(
+                label="Name",
+                placeholder="test",
+                autocomplete=options,
+                on_change=lambda e: result.set_text(e),
+            )
+            result = ui.label()
+
+            with ui.row():
+                ui.button("Close", on_click=dialog.close)
+                ui.button("Enter", on_click=lambda: on_record())
+
+            dialog.open()
+
+    def terminate():
+        global PROC
+        PROC.send_signal(signal.SIGINT)
+
+        # wait for process to terminate
+        PROC.wait()
+        ui.notify("Stopped recording")
+        record_button._props["name"] = "radio_button_checked"
+        record_button.on("click", lambda: recording_prompt(options, record_button))
+
+        PROC = None
+
+    def begin():
+        name = result.text.__getattribute__("value")
+
+        ui.notify(
+            f"Recording {name}... Press CTRL + C in terminal window to cancel",
+        )
+        PROC = Popen(
+            "python3 -m openadapt.record " + name,
+            shell=True,
+        )
+        record_button._props["name"] = "stop"
+        record_button.on("click", lambda: terminate())
+        record_button.update()
+        return PROC
+
+    def on_record():
+        global PROC
+        dialog.close()
+        PROC = begin()

openadapt/app/main.py

@@ -0,0 +1,82 @@
+import threading
+import base64
+import os
+
+from nicegui import app, ui
+
+from openadapt import replay, visualize
+from openadapt.app.cards import recording_prompt, select_import, settings
+from openadapt.app.util import clear_db, on_export, on_import
+from openadapt.app.objects.console import Console
+
+SERVER = "127.0.0.1:8000/upload"
+
+
+def run_app():
+    file = os.path.dirname(__file__)
+    app.native.window_args["resizable"] = False  # too many issues with resizing
+    app.native.start_args["debug"] = False
+
+    dark = ui.dark_mode()
+    logger = None
+
+    # Add logo
+    # right align icon
+    with ui.row().classes("w-full justify-right"):
+        # settings
+
+        # alignment trick
+        with ui.avatar(color="white" if dark else "black", size=128):
+            logo_base64 = base64.b64encode(open(f"{file}/assets/logo.png", "rb").read())
+            img = bytes(
+                f"data:image/png;base64,{(logo_base64.decode('utf-8'))}",
+                encoding="utf-8",
+            )
+            ui.image(img.decode("utf-8"))
+        ui.icon("settings").tooltip("Settings").on("click", lambda: settings(dark))
+        ui.icon("delete").on("click", lambda: clear_db(log=logger)).tooltip(
+            "Clear all recorded data"
+        )
+        ui.icon("upload").tooltip("Export Data").on("click", lambda: on_export(SERVER))
+        ui.icon("download").tooltip("Import Data").on(
+            "click", lambda: select_import(on_import)
+        )
+        ui.icon("share").tooltip("Share").on(
+            "click", lambda: (_ for _ in ()).throw(Exception(NotImplementedError))
+        )
+
+    # Recording description autocomplete
+    options = ["test"]
+
+    with ui.splitter(value=20) as splitter:
+        splitter.classes("w-full h-full")
+        with splitter.before:
+            with ui.column().classes("w-full h-full"):
+                record_button = (
+                    ui.icon("radio_button_checked", size="64px")
+                    .on("click", lambda: recording_prompt(options, record_button))
+                    .tooltip("Record a new replay / Stop recording")
+                )
+                ui.icon("visibility", size="64px").on(
+                    "click", lambda: threading.Thread(target=visualize.main).start()
+                ).tooltip("Visualize the latest replay")
+
+                ui.icon("play_arrow", size="64px").on(
+                    "click", lambda: replay.replay("NaiveReplayStrategy")
+                ).tooltip("Play the latest replay")
+        with splitter.after:
+            logger = Console()
+            logger.log.style("height: 250px;, width: 300px;")
+        splitter.enabled = False
+
+    ui.run(
+        title="OpenAdapt Client",
+        native=True,
+        window_size=(400, 400),
+        fullscreen=False,
+        reload=False,
+    )
+
+
+if __name__ == "__main__":
+    run_app()

openadapt/app/objects/console.py

@@ -0,0 +1,21 @@
+import sys
+
+from nicegui import ui
+
+
+class Console(object):
+    def __init__(self):
+        self.log = ui.log().classes("w-full h-20")
+        self.old_stderr = sys.stderr
+        sys.stderr = self
+
+    def write(self, data):
+        self.log.push(data[:-1])
+        self.log.update()
+
+    def flush(self):
+        self.log.update()
+
+    def reset(self):
+        self.log.clear()
+        sys.stderr = self.old_stderr