resolve merge conflicts

.gitignore

@@ -13,3 +13,14 @@ cache
 
 # db
 *.db
+*.db-journal
+
+# VSCode
+.VSCode
+.vsCode
+
+# Generated performance charts
+performance
+
+# Generated when adding editable dependencies in requirements.txt (-e)
+src

README.md

@@ -7,7 +7,7 @@ with the power of Transformers by:
 
 - Recording screenshots and associated user input
 - Aggregating and visualizing user input and recordings for development
-- Converting screenshots and user input into tokenized format
+- Converting screenshots and user input into tok/enized format
 - Generating synthetic input via transformer model completions
 - Replaying synthetic input to complete tasks
 
@@ -31,18 +31,23 @@ source .venv/bin/activate
 pip install wheel
 pip install -r requirements.txt
 pip install -e .
+python -m spacy download en_core_web_trf
 alembic upgrade head
 pytest
 ```
 
+## Permissions
+
+See how to set up system permissions on macOS [here](./permissions_in_macOS.md).
+
 ## Run
 
 ### Record
 
 Create a new recording by running the following command:
 
 ```
-python -m puterbot.record "testing out puterbot"
+python -m openadapt.record "testing out openadapt"
 ```
 
 Wait until all three event writers have started:
@@ -55,30 +60,34 @@ Wait until all three event writers have started:
 Type a few words into the terminal and move your mouse around the screen
 to generate some events, then stop the recording by pressing CTRL+C.
 
-Note: keep your recording short (i.e. under a minute), as they are
+Current limitations: 
+- recording should be short (i.e. under a minute), as they are
 somewhat memory intensive, and there is currently an
 [open issue](https://github.com/MLDSAI/OpenAdapt/issues/5) describing a
-possible memory leak.
+possible memory leak
+- the only touchpad and trackpad gestures currently supported are 
+pointing the cursor and left or right clicking, as described in this
+[open issue](https://github.com/MLDSAI/OpenAdapt/issues/145)
 
 
 ### Visualize
 
 Visualize the latest recording you created by running the following command:
 
 ```
-python -m puterbot.visualize
+python -m openadapt.visualize
 ```
 
 This will open your browser. It will look something like this:
 
-![visualize.py](./assets/visualize.png)
+![image](https://github.com/MLDSAI/OpenAdapt/assets/774615/5d7253b7-ae12-477c-94a3-b388e4f37587)
 
 ### Playback
 
 You can play back the recording using the following command:
 
 ```
-python -m puterbot.replay NaiveReplayStrategy
+python -m openadapt.replay NaiveReplayStrategy
 ```
 
 More ReplayStrategies coming soon! (see [Contributing](#Contributing)).
@@ -88,8 +97,6 @@ More ReplayStrategies coming soon! (see [Contributing](#Contributing)).
 
 ### Design
 
-![image](https://user-images.githubusercontent.com/774615/236658984-01f9c06b-d132-40ee-a716-205fa76bf3f2.png)
-
 ### Problem Statement
 
 Our goal is to automate the task described and demonstrated in a `Recording`.
@@ -176,25 +183,18 @@ If you're interested in getting paid for your work, please mention it in your Pu
 
 ## Troubleshooting
 
-Apple Silicon:
+MacOS: if you encounter system alert messages or find issues when making and replaying recordings, make sure to [set up permissions accordingly](./permissions_in_macOS.md).
 
-```
-$ python puterbot/record.py
-...
-This process is not trusted! Input event monitoring will not be possible until it is added to accessibility clients.
-```
+![MacOS System Alerts](./assets/macOS_permissions_alert.png)
 
-Solution:
-https://stackoverflow.com/a/69673312
+In summary (from https://stackoverflow.com/a/69673312):
 
-```
-Settings -> Security & Privacy
-Click on the Privacy tab
-Scroll and click on the Accessibility Row
-Click the +
-Navigate to /System/Applications/Utilities/ or wherever the Terminal.app is installed
-Click okay.
-```
+1. Settings -> Security & Privacy
+2. Click on the Privacy tab
+3. Scroll and click on the Accessibility Row
+4. Click +
+5. Navigate to /System/Applications/Utilities/ (or wherever Terminal.app is installed)
+6. Click okay.
 
 ## Developing
 
@@ -206,7 +206,7 @@ alembic revision --autogenerate -m "<msg>"
 
 # Submitting an Issue
 
-Please submit any issues to https://github.com/MLDSAI/puterbot/issues with the
+Please submit any issues to https://github.com/MLDSAI/openadapt/issues with the
 following information:
 
 - Problem description (please include any relevant console output and/or screenshots)

alembic/env.py

@@ -7,20 +7,21 @@
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
-config = context.config
+alembic_config = context.config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
+if alembic_config .config_file_name is not None:
+    fileConfig(alembic_config .config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from puterbot.config import DB_URL
-from puterbot.models import *
-from puterbot.db import Base
+from openadapt.config import DB_URL
+from openadapt.models import *
+from openadapt.db import Base
+
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
@@ -46,7 +47,7 @@ def run_migrations_offline() -> None:
     script output.
 
     """
-    #url = config.get_main_option("sqlalchemy.url")
+    # url = config.get_main_option("sqlalchemy.url")
     url = get_url()
     context.configure(
         url=url,
@@ -67,7 +68,7 @@ def run_migrations_online() -> None:
     and associate a connection with the context.
 
     """
-    configuration = config.get_section(config.config_ini_section)
+    configuration = alembic_config .get_section(alembic_config .config_ini_section)
     configuration["sqlalchemy.url"] = get_url()
     connectable = engine_from_config(
         configuration=configuration,

alembic/versions/104d4a614d95_add_performancestat.py

@@ -0,0 +1,36 @@
+"""add PerformanceStat
+
+Revision ID: 104d4a614d95
+Revises: b2dc41850120
+Create Date: 2023-05-27 02:59:14.032373
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '104d4a614d95'
+down_revision = 'b2dc41850120'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('performance_stat',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('recording_timestamp', sa.Integer(), nullable=True),
+    sa.Column('event_type', sa.String(), nullable=True),
+    sa.Column('start_time', sa.Integer(), nullable=True),
+    sa.Column('end_time', sa.Integer(), nullable=True),
+    sa.Column('window_id', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_performance_stat'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('performance_stat')
+    # ### end Alembic commands ###

alembic/versions/57d78d23087a_add_windowevent_state.py

@@ -0,0 +1,32 @@
+"""add WindowEvent.state
+
+Revision ID: 57d78d23087a
+Revises: 20f9c2afb42c
+Create Date: 2023-05-14 18:32:57.473479
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '57d78d23087a'
+down_revision = '20f9c2afb42c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('state', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_column('state')
+
+    # ### end Alembic commands ###

alembic/versions/b2dc41850120_window_window_id.py

@@ -0,0 +1,32 @@
+"""Window.window_id
+
+Revision ID: b2dc41850120
+Revises: d63569e4fb90
+Create Date: 2023-05-17 12:50:35.125610
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b2dc41850120'
+down_revision = 'd63569e4fb90'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('window_id', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_column('window_id')
+
+    # ### end Alembic commands ###

alembic/versions/d63569e4fb90_actionevent_element_state.py

@@ -0,0 +1,32 @@
+"""ActionEvent.element_state
+
+Revision ID: d63569e4fb90
+Revises: ec337f277666
+Create Date: 2023-05-16 21:43:00.120143
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd63569e4fb90'
+down_revision = 'ec337f277666'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('action_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('element_state', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('action_event', schema=None) as batch_op:
+        batch_op.drop_column('element_state')
+
+    # ### end Alembic commands ###

alembic/versions/ec337f277666_datetime_timestamp.py

@@ -0,0 +1,38 @@
+"""DateTime timestamp
+
+Revision ID: ec337f277666
+Revises: 57d78d23087a
+Create Date: 2023-05-16 17:51:00.061604
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec337f277666'
+down_revision = '57d78d23087a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('screenshot', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_screenshot_recording_timestamp_recording'), 'recording', ['recording_timestamp'], ['timestamp'])
+
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_window_event_recording_timestamp_recording'), 'recording', ['recording_timestamp'], ['timestamp'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_window_event_recording_timestamp_recording'), type_='foreignkey')
+
+    with op.batch_alter_table('screenshot', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_screenshot_recording_timestamp_recording'), type_='foreignkey')
+
+    # ### end Alembic commands ###