Add Chrome browser event in database during recording

README.md

@@ -164,6 +164,40 @@ possible memory leak
 pointing the cursor and left or right clicking, as described in this
 [open issue](https://github.com/OpenAdaptAI/OpenAdapt/issues/145)
 
+
+### Capturing Browser Events
+
+To capture (record) browser events on Chrome, follow the following steps:
+
+1. Go To: [Chrome Extension Page](chrome://extensions/)
+
+2. Turn on the `Developer mode` to on (near top right):
+
+![image](https://github.com/OpenAdaptAI/OpenAdapt/assets/65433817/c97eb9fb-05d6-465d-85b3-332694556272)
+
+3. Click on `Load Unpacked` (near top left).
+
+![image](https://github.com/OpenAdaptAI/OpenAdapt/assets/65433817/00c8adf5-074a-4655-b132-fd87644007fc)
+
+4. Select `chrome_extension` directory:
+
+![image](https://github.com/OpenAdaptAI/OpenAdapt/assets/65433817/71610ed3-f8d4-431a-9a22-d901127b7b0c)
+
+5. You shall see this:
+
+![image](https://github.com/OpenAdaptAI/OpenAdapt/assets/65433817/7ee19da9-37e0-448f-b9ab-08ef99110e85)
+
+Meaning that the extension is loaded !
+
+6. Switch the flag to true (if it false):
+
+![image](https://github.com/user-attachments/assets/8eba24a3-7c68-4deb-8fbe-9d03cece1482)
+
+7. Finally, start recording, once Everything is started, then go to chrome browser and search for some page, do a few click and then stop the recording. Let it finish successfully.
+
+8. Once done, you can check the `openadapt.db` -> table `browser_event` it should have all of your browser activity logs in it. You can check for correctness by using `sqlite3` cli or using an extension like SQLite Viewer in VS Code to open `openadapt.db`.
+
+
 ### Visualize
 
 Quickly visualize the latest recording you created by running the following command:

chrome_extension/background.js

@@ -0,0 +1,49 @@
+/**
+ * @file background.js
+ * @description Creates a new background script that listens for messages from the content script
+ * and sends them to a WebSocket server.
+*/
+
+let socket;
+let TIMEOUT_INTERVAL = 5000; // ms
+
+/*
+ * Function to connect to the WebSocket server.
+*/
+function connectWebSocket() {
+  // TODO: Ideally we read "ws://localhost:8765" and `TIMEOUT_INTERVAL` from config.py, or it gets passed in somehow. 
+  socket = new WebSocket("ws://localhost:8765");
+
+  socket.onopen = function() {
+    console.log("WebSocket connection established");
+  };
+
+  socket.onmessage = function(event) {
+    console.log("Message from server:", event.data);
+  };
+
+  socket.onclose = function(event) {
+    console.log("WebSocket connection closed", event);
+    // Reconnect after 5 seconds if the connection is lost
+    setTimeout(connectWebSocket, TIMEOUT_INTERVAL);
+  };
+
+  socket.onerror = function(error) {
+    console.error("WebSocket error:", error);
+    socket.close();
+  };
+}
+
+// Create a connection to the WebSocket server
+connectWebSocket();
+
+/* Listen for messages from the content script */
+chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
+  if (socket && socket.readyState === WebSocket.OPEN) {
+    console.log(message);
+    socket.send(JSON.stringify(message));
+    sendResponse({status: "Message sent to WebSocket"});
+  } else {
+    sendResponse({status: "WebSocket connection not open"});
+  }
+});

chrome_extension/content.js

@@ -0,0 +1,156 @@
+/**
+ * @file content.js
+ * @description This file is injected into the web page and is responsible for
+ * capturing DOM changes and sending them to the background script.
+*/
+
+
+const elements = {};
+
+
+/*
+ * Function to send a message to the background script
+*/
+function sendMessageToBackgroundScript(message) {
+  chrome.runtime.sendMessage(message);
+}
+
+
+/*
+ * Function to capture initial document state and
+ * send it to the background script
+*/
+function captureDocumentState() {
+  const documentBody = document.body.outerHTML;
+  const documentHead = document.head.outerHTML;
+  const page_url = window.location.href;
+
+  sendMessageToBackgroundScript({
+    action: "captureDocumentState",
+    documentBody: documentBody,
+    documentHead: documentHead,
+    elements: elements,
+    url: page_url,
+    timestamp: Date.now(),
+  });
+}
+
+
+/*
+ * Function to handle click events on any element on a web page.
+ * It sends the element details to the background script.
+*/
+function handleElementClick(event) {
+  const element = event.target;
+  const tagName = element.tagName;
+  const { x, y } = elements[element.id] || {};
+  const value = elements[element.id]?.value || "";
+  const attributes = {};
+
+  for (const attr of element.attributes) {
+    attributes[attr.name] = attr.value;
+  }
+
+  sendMessageToBackgroundScript({
+    action: "elementClicked",
+    tagName: tagName,
+    attributes: attributes,
+    x: x,
+    y: y,
+    value: value,
+    url: window.location.href,
+    timestamp: Date.now(),
+  });
+}
+
+
+/*
+ * Function to create a debounced version of a function.
+ * @param {Function} func - The function to be debounced.
+ * @param {number} delay - The delay in milliseconds.
+*/
+function debounce(func, delay) {
+  let timerId;
+  return function (...args) {
+    if (timerId) {
+      clearTimeout(timerId);
+    }
+    timerId = setTimeout(() => {
+      func.apply(this, args);
+      timerId = null;
+    }, delay);
+  };
+}
+
+
+/*
+  * Function to handle input events on any element on a web page.
+  * It sends the element details to the background script.
+  * @param {Event} event - The input event.
+  * @param {Element} element - The element on which the input event occurred.
+*/
+function handleDebouncedInput(event) {
+  const element = event.target;
+  const { x, y } = elements[element.id];
+  const value = elements[element.id].element.value;
+  const tagName = element.tagName;
+  const attributes = {};
+
+  for (const attr of element.attributes) {
+    attributes[attr.name] = attr.value;
+  }
+
+  sendMessageToBackgroundScript({
+    action: "elementInput",
+    tagName: tagName,
+    attributes: attributes,
+    x: x,
+    y: y,
+    value: value,
+    url: window.location.href,
+    timestamp: Date.now(),
+  });
+}
+
+/* Debounce Input Handler */
+const debouncedInputHandler = debounce(handleDebouncedInput, 500);
+
+/* Call Debounce Input Handler */
+function handleElementInput(event) {
+  debouncedInputHandler(event);
+}
+
+
+/*
+ * Add an click and input information of the element.
+*/
+function addElement(element) {
+  const rect = element.getBoundingClientRect();
+  const x = rect.left + window.scrollX;
+  const y = rect.top + window.scrollY;
+  const value = element.value;
+  if (!element.id) {
+    element.id = element.tagName + "_" + x + "_" + y;
+  }
+  elements[element.id] = { element, x, y, value };
+  element.addEventListener("click", handleElementClick);
+  element.addEventListener("input", debounce(handleDebouncedInput, 500));
+}
+
+
+/*
+ * Handle all events that occur on the page.
+*/
+function addEventListeners() {
+  const elements = document.getElementsByTagName("*");
+
+  for (const element of elements) {
+    addElement(element);
+  }
+}
+
+
+/* Function Calls */
+
+addEventListeners(); // Adds listeners
+captureDocumentState(); // Captures initial document state

chrome_extension/manifest.json

@@ -0,0 +1,23 @@
+{
+  "name": "openadapt",
+  "description": "Native messaging example add-on",
+  "version": "1.0",
+  "manifest_version": 3,
+  "icons": {
+    "48": "icons/logo.png"
+  },
+  "action": {
+    "default_icon": "icons/logo.png"
+  },
+  "background": {
+    "service_worker": "background.js"
+  },
+  "permissions": ["activeTab", "tabs", "nativeMessaging", "scripting"],
+  "host_permissions": ["<all_urls>"],
+  "content_scripts": [
+    {
+      "matches": ["<all_urls>"],
+      "js": ["content.js"]
+    }
+  ]
+}

openadapt/alembic/versions/113f15863a33_regenrate_with_browser_event_table.py

@@ -0,0 +1,43 @@
+"""regenrate with browser_event table
+
+Revision ID: 113f15863a33
+Revises: bb25e889ad71
+Create Date: 2024-07-08 19:59:44.713853
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+import openadapt
+
+
+# revision identifiers, used by Alembic.
+revision = '113f15863a33'
+down_revision = 'bb25e889ad71'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('replay',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True),
+    sa.Column('strategy_name', sa.String(), nullable=True),
+    sa.Column('strategy_args', sa.JSON(), nullable=True),
+    sa.Column('git_hash', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_replay'))
+    )
+    with op.batch_alter_table('action_event', schema=None) as batch_op:
+        batch_op.drop_constraint('fk_input_event_browser_event_timestamp_browser_event', type_='foreignkey')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('action_event', schema=None) as batch_op:
+        batch_op.create_foreign_key('fk_input_event_browser_event_timestamp_browser_event', 'browser_event', ['browser_event_timestamp'], ['timestamp'])
+
+    op.drop_table('replay')
+    # ### end Alembic commands ###

openadapt/alembic/versions/b206c80f7640_add_recording_inputevent_screenshot_.py

@@ -53,6 +53,14 @@ def upgrade() -> None:
         sa.Column("height", sa.Integer(), nullable=True),
         sa.PrimaryKeyConstraint("id", name=op.f("pk_window_event")),
     )
+    op.create_table(
+        "browser_event",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("recording_timestamp", sa.Integer(), nullable=True),
+        sa.Column("timestamp", sa.Integer(), nullable=True),
+        sa.Column("message", sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_browser_event")),
+    )
     op.create_table(
         "input_event",
         sa.Column("id", sa.Integer(), nullable=False),
@@ -61,6 +69,7 @@ def upgrade() -> None:
         sa.Column("recording_timestamp", sa.Integer(), nullable=True),
         sa.Column("screenshot_timestamp", sa.Integer(), nullable=True),
         sa.Column("window_event_timestamp", sa.Integer(), nullable=True),
+        sa.Column("browser_event_timestamp", sa.Integer(), nullable=True),
         sa.Column("mouse_x", sa.Numeric(asdecimal=False), nullable=True),
         sa.Column("mouse_y", sa.Numeric(asdecimal=False), nullable=True),
         sa.Column("mouse_dx", sa.Numeric(asdecimal=False), nullable=True),
@@ -94,6 +103,11 @@ def upgrade() -> None:
             ["window_event.timestamp"],
             name=op.f("fk_input_event_window_event_timestamp_window_event"),
         ),
+        sa.ForeignKeyConstraint(
+            ["browser_event_timestamp"],
+            ["browser_event.timestamp"],
+            name=op.f("fk_input_event_browser_event_timestamp_browser_event"),
+        ),
         sa.PrimaryKeyConstraint("id", name=op.f("pk_input_event")),
     )
     # ### end Alembic commands ###
@@ -103,6 +117,7 @@ def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table("input_event")
     op.drop_table("window_event")
+    op.drop_table("browser_event")
     op.drop_table("screenshot")
     op.drop_table("recording")
     # ### end Alembic commands ###