Merge branch 'main' into publishing-with-pypi

.gitignore

@@ -13,3 +13,12 @@ cache
 
 # db
 *.db
+
+# VSCode
+.VSCode
+
+# Generated performance charts
+performance
+
+# Generated when adding editable dependencies in requirements.txt (-e)
+src

README.md

@@ -8,7 +8,7 @@ with the power of Transformers by:
 
 - Recording screenshots and associated user input
 - Aggregating and visualizing user input and recordings for development
-- Converting screenshots and user input into tokenized format
+- Converting screenshots and user input into tok/enized format
 - Generating synthetic input via transformer model completions
 - Replaying synthetic input to complete tasks
 
@@ -36,6 +36,10 @@ alembic upgrade head
 pytest
 ```
 
+## Permissions
+
+See how to set up system permissions on macOS [here](./permissions_in_macOS.md).
+
 ## Run
 
 ### Record
@@ -76,7 +80,8 @@ python -m openadapt.visualize
 
 This will open your browser. It will look something like this:
 
-![visualize.py](https://github-production-user-asset-6210df.s3.amazonaws.com/43456930/241332686-dd490362-064f-4b47-b65f-1ba31f71fa95.png)
+![image](https://github.com/MLDSAI/OpenAdapt/assets/774615/5d7253b7-ae12-477c-94a3-b388e4f37587)
+
 
 ### Playback
 
@@ -93,8 +98,6 @@ More ReplayStrategies coming soon! (see [Contributing](#Contributing)).
 
 ### Design
 
-![image](https://user-images.githubusercontent.com/774615/236658984-01f9c06b-d132-40ee-a716-205fa76bf3f2.png)
-
 ### Problem Statement
 
 Our goal is to automate the task described and demonstrated in a `Recording`.
@@ -181,25 +184,18 @@ If you're interested in getting paid for your work, please mention it in your Pu
 
 ## Troubleshooting
 
-Apple Silicon:
+MacOS: if you encounter system alert messages or find issues when making and replaying recordings, make sure to [set up permissions accordingly](./permissions_in_macOS.md).
 
-```
-$ python openadapt/record.py
-...
-This process is not trusted! Input event monitoring will not be possible until it is added to accessibility clients.
-```
+![MacOS System Alerts](./assets/macOS_permissions_alert.png)
 
-Solution:
-https://stackoverflow.com/a/69673312
+In summary (from https://stackoverflow.com/a/69673312):
 
-```
-Settings -> Security & Privacy
-Click on the Privacy tab
-Scroll and click on the Accessibility Row
-Click the +
-Navigate to /System/Applications/Utilities/ or wherever the Terminal.app is installed
-Click okay.
-```
+1. Settings -> Security & Privacy
+2. Click on the Privacy tab
+3. Scroll and click on the Accessibility Row
+4. Click +
+5. Navigate to /System/Applications/Utilities/ (or wherever Terminal.app is installed)
+6. Click okay.
 
 ## Developing
 

alembic/versions/104d4a614d95_add_performancestat.py

@@ -0,0 +1,36 @@
+"""add PerformanceStat
+
+Revision ID: 104d4a614d95
+Revises: b2dc41850120
+Create Date: 2023-05-27 02:59:14.032373
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '104d4a614d95'
+down_revision = 'b2dc41850120'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('performance_stat',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('recording_timestamp', sa.Integer(), nullable=True),
+    sa.Column('event_type', sa.String(), nullable=True),
+    sa.Column('start_time', sa.Integer(), nullable=True),
+    sa.Column('end_time', sa.Integer(), nullable=True),
+    sa.Column('window_id', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_performance_stat'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('performance_stat')
+    # ### end Alembic commands ###

install/install_openadapt.ps1

@@ -0,0 +1,177 @@
+# PowerShell script to pull OpenAdapt and install
+
+# Change these if a different version of is required
+$pythonCmd = "python3.10"
+$pythonVerStr = "Python 3.10*"
+$pythonInstaller = "python-3.10.11-amd64.exe"
+$pythonInstallerLoc = "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe"
+
+$gitInstaller = "Git-2.40.1-64-bit.exe"
+$gitInstallerLoc = "https://github.com/git-for-windows/git/releases/download/v2.40.1.windows.1/Git-2.40.1-64-bit.exe"
+$gitUninstaller = "C:\Program Files\Git\unins000.exe"
+
+$VCRedistInstaller = "vc_redist.x64.exe"
+$VCRedistInstallerLoc = "https://aka.ms/vs/17/release/vc_redist.x64.exe"
+$VCRedistRegPath = "HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\X64"
+
+
+# Return true if a command/exe is available
+function CheckCMDExists() {
+    Param
+    (
+        [Parameter(Mandatory = $true)] [string] $command
+    )
+
+    Get-Command $command -errorvariable getErr -erroraction 'silentlycontinue'
+    If ($getErr -eq $null) {
+        $true;
+    }
+    return $false;
+}
+
+# Get command for python, install python if required version is unavailable
+function GetPythonCMD() {
+    # Use python alias of required version if it exists
+    if (CheckCMDExists($pythonCmd)) {
+        return $pythonCmd
+    }
+
+    # Use python exe if it exists and is the required version
+    $pythonCmd = "python"
+    if (CheckCMDExists($pythonCmd)) {
+        $res = Invoke-Expression "python -V"
+        if ($res -like $pythonVerStr) {
+            return $pythonCmd
+        }
+    }
+
+    # Install required python version
+    Write-Host "Downloading python installer"
+    $ProgressPreference = 'SilentlyContinue'
+    Invoke-WebRequest -Uri $pythonInstallerLoc -OutFile $pythonInstaller
+    $exists = Test-Path -Path $pythonInstaller -PathType Leaf
+    if(!$exists) {
+        Write-Host "Failed to download python installer"
+        Cleanup
+        exit
+    }
+
+    Write-Host "Installing python, click 'Yes' if prompted for permission"
+    Start-Process -FilePath $pythonInstaller -Verb runAs -ArgumentList '/quiet','InstallAllUsers=0','PrependPath=1' -Wait
+
+    #Refresh Path Environment Variable
+    $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+
+    # Make sure python is now available and the right version
+    if (CheckCMDExists($pythonCmd)) {
+        $res = Invoke-Expression "python -V"
+        if ($res -like $pythonVerStr) {
+            Remove-Item $pythonInstaller
+            return $pythonCmd
+        }
+    }
+
+    Write-Host "Error after installing python. Uninstalling, click 'Yes' if prompted for permission"
+    Start-Process -FilePath $pythonInstaller -Verb runAs -ArgumentList '/quiet','/uninstall' -Wait
+    Remove-Item $pythonInstaller
+
+    # Stop OpenAdapt install
+    Cleanup
+    exit
+}
+
+function Cleanup {
+    $exists = Test-Path -Path "..\OpenAdapt"
+    if($exists) {
+        Set-Location ..\
+        Remove-Item -LiteralPath "OpenAdapt" -Force -Recurse
+    }
+}
+
+# Run a command and ensure it did not fail
+function RunAndCheck {
+    Param
+    (
+        [Parameter(Mandatory = $true)] [string] $Command,
+        [Parameter(Mandatory = $true)] [string] $Desc
+    )
+
+    Invoke-Expression $Command
+    if ($LastExitCode) {
+        Write-Host "Failed: $Desc : $LastExitCode"
+        Cleanup
+        exit
+    }
+
+    Write-Host "Success: $Desc"
+}
+
+$gitExists = CheckCMDExists "git"
+if (!$gitExists) {
+    # Install git
+    Write-Host "Downloading git installer"
+    $ProgressPreference = 'SilentlyContinue'
+    Invoke-WebRequest -Uri $gitInstallerLoc -OutFile $gitInstaller
+    $exists = Test-Path -Path $gitInstaller -PathType Leaf
+    if(!$exists) {
+        Write-Host "Failed to download git installer"
+        exit
+    }
+
+    Write-Host "Installing git, click 'Yes' if prompted for permission"
+    Start-Process -FilePath $gitInstaller -Verb runAs -ArgumentList '/VERYSILENT /NORESTART /NOCANCEL /SP- /CLOSEAPPLICATIONS /RESTARTAPPLICATIONS /COMPONENTS="icons,ext\reg\shellhere,assoc,assoc_sh"' -Wait
+    Remove-Item $gitInstaller
+
+    #Refresh Path Environment Variable
+    $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+
+    # Make sure git is now available
+    $gitExists = CheckCMDExists "git"
+    if (!$gitExists) {
+        Write-Host "Error after installing git. Uninstalling, click 'Yes' if prompted for permission"
+        Start-Process -FilePath $gitUninstaller -Verb runAs -ArgumentList '/VERYSILENT','/SUPPRESSMSGBOXES','/NORESTART' -Wait
+        exit
+    }
+}
+
+# Check if Visual C++ Redist is installed
+# Note: Temporarily setting $ErrorActionPreference as -erroraction 'silentlycontinue' doesn't prevent non-terminating error with Get-ItemPropertyValue
+$ErrorActionPreference="SilentlyContinue"
+$vcredistExists = Get-ItemPropertyValue -Path $VCRedistRegPath -erroraction 'silentlycontinue' -Name Installed
+$ErrorActionPreference="Continue"
+
+if (!$vcredistExists) {
+    # Install Visual C++ Redist
+    Write-Host "Downloading Visual C++ Redist"
+    $ProgressPreference = 'SilentlyContinue'
+    Invoke-WebRequest -Uri $VCRedistInstallerLoc -OutFile $VCRedistInstaller
+    $exists = Test-Path -Path $VCRedistInstaller -PathType Leaf
+    if(!$exists) {
+        Write-Host "Failed to download Visual C++ installer"
+        exit
+    }
+
+    Write-Host "Installing Visual C++ Redist, click 'Yes' if prompted for permission"
+    Start-Process -FilePath $VCRedistInstaller -Verb runAs -ArgumentList "/install /q /norestart" -Wait
+    Remove-Item $VCRedistInstaller
+
+    if($LastExitCode) {
+        "Failed to install Visual C++ Redist: $LastExitCode"
+        exit
+    }
+}
+
+RunAndCheck "git clone -q https://github.com/MLDSAI/OpenAdapt.git" "clone git repo"
+
+Set-Location .\OpenAdapt
+
+$python = GetPythonCMD
+
+RunAndCheck "$python -m venv .venv" "create python virtual environment"
+RunAndCheck ".venv\Scripts\Activate.ps1" "enable python virtual environment"
+RunAndCheck "pip install wheel" "pip install wheel"
+RunAndCheck "pip install -r requirements.txt" "pip install -r requirements.txt"
+RunAndCheck "pip install -e ." "pip install -e ."
+RunAndCheck "alembic upgrade head" "alembic upgrade head"
+RunAndCheck "python -m spacy download en_core_web_trf" "python -m spacy download en_core_web_trf"
+RunAndCheck "pytest" "run OpenAdapt tests"

install/install_openadapt.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+# Run a command and ensure it did not fail
+RunAndCheck() {
+    res=$($1)
+
+    if [[ -z $res ]] || [[ $res == 0 ]]; then
+        echo "Success: $2 : $res"
+    else
+        echo "Failed: $2 : $res"
+        exit 1
+    fi
+}
+[ -d "OpenAdapt" ] && mv OpenAdapt OpenAdapt-$(date +%Y-%m-%d_%H-%M-%S)
+RunAndCheck "git clone https://github.com/MLDSAI/OpenAdapt.git" "clone git repo"
+
+cd OpenAdapt
+
+RunAndCheck "python3.10 -m venv .venv" "create python virtual environment"
+source .venv/bin/activate
+pip install wheel
+
+# the following line generates a warning:
+#   Ignoring pywin32: markers 'sys_platform == "win32"' don't match your environment
+pip install -r requirements.txt
+
+# the following line generates a warning:
+#   [notice] To update, run: pip install --upgrade pip
+pip install -e .
+
+RunAndCheck "alembic upgrade head"
+RunAndCheck "python3.10 -m spacy download en_core_web_trf"
+RunAndCheck "pytest"

openadapt/config.py

@@ -12,7 +12,7 @@
     "CACHE_VERBOSITY": 0,
     "DB_ECHO": False,
     "DB_FNAME": "openadapt.db",
-    "OPENAI_API_KEY": None,
+    "OPENAI_API_KEY": "<set your api key in .env>",
     #"OPENAI_MODEL_NAME": "gpt-4",
     "OPENAI_MODEL_NAME": "gpt-3.5-turbo",
     # may incur significant performance penalty