diff --git a/.github/ISSUE_TEMPLATE/bug_form.yml b/.github/ISSUE_TEMPLATE/bug_form.yml new file mode 100644 index 000000000..8daf424e9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_form.yml @@ -0,0 +1,58 @@ +name: Bug Report +description: File a bug report +title: "[Bug]: " +labels: bug +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: input + id: describe-bug + attributes: + label: Describe the bug + description: What behaviour was expected and what actually happened? + placeholder: null + validations: + required: true + - type: textarea + id: steps + attributes: + label: To Reproduce + description: What steps would we take to reproduce the behaviour + placeholder: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be + automatically formatted into code, so no need for backticks. + render: shell + - type: textarea + id: image-upload + attributes: + label: Upload Screenshot + description: Please upload an image that helps illustrate the issue. + validations: + required: false + - type: dropdown + id: OS + attributes: + label: What Operating System did you use? + multiple: true + options: + - macOS + - Apple iOS + - Microsoft Windows + - Google Android + - Linux + - Other + validations: + required: true + - type: textarea + id: other + attributes: + label: Additional Context + description: null diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..ac09e8636 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +blank_issues_enabled: true +version: 0.1 diff --git a/.github/ISSUE_TEMPLATE/new_feature.yml b/.github/ISSUE_TEMPLATE/new_feature.yml new file mode 100644 index 000000000..127573f7e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_feature.yml @@ -0,0 +1,28 @@ +name: "Feature Request" +description: Submit a proposal for a new OpenAdapt feature +labels: enhancement +body: + - type: textarea + id: feature-request + validations: + required: true + attributes: + label: Feature request + description: A clear and concise description of the feature proposal. Please provide links to any relevant resources. + + - type: textarea + id: motivation + validations: + required: true + attributes: + label: Motivation + description: Please outline the purpose for the proposal (e.g., is it related to a problem?). Add any relevant links (e.g. GotHub issues). + placeholder: I'm always frustrated when [...] so this feature would [...]. + + - type: textarea + id: other + validations: + required: false + attributes: + label: Other + description: Any additional details diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..d7e5d06cc --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ + + +**What kind of change does this PR introduce?** + + + +**Checklist** +* [ ] My code follows the style guidelines of this project +* [ ] I have pefomed a self-review of my code +* [ ] If applicable, I have added tests to prove my fix is functional/effective +* [ ] I have linted my code locally prior to submission +* [ ] I have commented my code, particularly in hard-to-understand areas +* [ ] I have made corresponding changes to the documentation (e.g. README.md, requirements.txt) +* [ ] New and existing unit tests pass locally with my changes + +**Summary** + + + +**How can your code be run and tested?** + + + + +**Other information** \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..b1088661a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,33 @@ +# How to contribute + +We would love to implement your contributions to this project! We simply ask that you observe the following guidelines. + +## Code Style + +This project follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html). + + +## Creating Issue +In order to effectively communicate any bugs or request new features, please select the appropriate form. If none of the options suit your needs, you can click on "Open a blank issue" located at the bottom. + +## Testing +[GitHub Actions](https://github.com/MLDSAI/OpenAdapt/actions/new) are automatically run on each pull request to ensure consistent behaviour and style. The Actions are composed of PyTest, [black](https://github.com/psf/black) and [flake8](https://flake8.pycqa.org/en/latest/user/index.html). + +You can run these tests on your own computer by downloading the depencencies in requirements.txt and then running pytest in the root directory. + +## Pull Request Format + +To speed up the review process, please use the provided pull request template and create a draft pull request to get initial feedback. + +The pull request template includes areas to explain the changes, and a checklist with boxes for code style, testing, and documenttation. + +## Submitting Changes + +1. Fork the current repository +2. Make a branch to work on, or use the main branch +3. Push desired changes onto the branch in step 2 +4. Submit a pull request with the branch in step 2 as the head ref and the MLDSAI/OpenAdapt main as the base ref + - Note: may need to click "compare across forks" +5. Update pull request using feedback + - this step may not be necessary, or may need to be repeated +6. Celebrate contributing to OpenAdapt! diff --git a/install/install_openadapt.ps1 b/install/install_openadapt.ps1 index 3732659e7..ee1370b6d 100644 --- a/install/install_openadapt.ps1 +++ b/install/install_openadapt.ps1 @@ -14,6 +14,7 @@ $VCRedistInstaller = "vc_redist.x64.exe" $VCRedistInstallerLoc = "https://aka.ms/vs/17/release/vc_redist.x64.exe" $VCRedistRegPath = "HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\X64" +# TODO: Add Tesseract OCR installation: https://tesseract-ocr.github.io/tessdoc/Installation.html # Return true if a command/exe is available function CheckCMDExists() { diff --git a/install/install_openadapt.sh b/install/install_openadapt.sh index 69c365fa1..7431df850 100644 --- a/install/install_openadapt.sh +++ b/install/install_openadapt.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +# TODO: Add Tesseract OCR installation: https://tesseract-ocr.github.io/tessdoc/Installation.html + # Run a command and ensure it did not fail RunAndCheck() { res=$($1) diff --git a/openadapt/config.py b/openadapt/config.py index 3dd21c777..2030538cc 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -30,10 +30,58 @@ "RECORD_READ_ACTIVE_ELEMENT_STATE": False, # TODO: remove? "REPLAY_STRIP_ELEMENT_STATE": True, + # IGNORES WARNINGS (PICKLING, ETC.) + "IGNORE_WARNINGS": False, # ACTION EVENT CONFIGURATIONS "ACTION_TEXT_SEP": "-", "ACTION_TEXT_NAME_PREFIX": "<", - "ACTION_TEXT_NAME_SUFFIX": ">" + "ACTION_TEXT_NAME_SUFFIX": ">", + # SCRUBBING CONFIGURATIONS + "SCRUB_ENABLED": True, + "SCRUB_CHAR": "*", + "SCRUB_LANGUAGE": "en", + # TODO support lists in getenv_fallback + "SCRUB_FILL_COLOR": (255, 0, 0), + "SCRUB_CONFIG_TRF": { + "nlp_engine_name": "spacy", + "models": [{"lang_code": "en", "model_name": "en_core_web_trf"}], + }, + "SCRUB_IGNORE_ENTITIES": [ + # 'US_PASSPORT', + # 'US_DRIVER_LICENSE', + # 'CRYPTO', + # 'UK_NHS', + # 'PERSON', + # 'CREDIT_CARD', + # 'US_BANK_NUMBER', + # 'PHONE_NUMBER', + # 'US_ITIN', + # 'AU_ABN', + "DATE_TIME", + # 'NRP', + # 'SG_NRIC_FIN', + # 'AU_ACN', + # 'IP_ADDRESS', + # 'EMAIL_ADDRESS', + "URL", + # 'IBAN_CODE', + # 'AU_TFN', + # 'LOCATION', + # 'AU_MEDICARE', + # 'US_SSN', + # 'MEDICAL_LICENSE' + ], + "SCRUB_KEYS_HTML": [ + "text", + "canonical_text", + "title", + "state", + "task_description", + "key_char", + "canonical_key_char", + "key_vk", + "children", + ], } @@ -62,53 +110,21 @@ def getenv_fallback(var_name): logger.info(f"{key}={val}") -# SCRUBBING CONFIGURATIONS -SCRUB_ENABLED = True -SCRUB_CHAR = "*" -SCRUB_LANGUAGE = "en" -SCRUB_CONFIG_TRF = { - "nlp_engine_name": "spacy", - "models": [ - { - "lang_code": "en", - "model_name": "en_core_web_trf" - } - ], -} -DEFAULT_SCRUB_FILL_COLOR = (255,0,0) -SCRUB_IGNORE_ENTITIES = [ - # 'US_PASSPORT', - # 'US_DRIVER_LICENSE', - # 'CRYPTO', - # 'UK_NHS', - # 'PERSON', - # 'CREDIT_CARD', - # 'US_BANK_NUMBER', - # 'PHONE_NUMBER', - # 'US_ITIN', - # 'AU_ABN', - "DATE_TIME", - # 'NRP', - # 'SG_NRIC_FIN', - # 'AU_ACN', - # 'IP_ADDRESS', - # 'EMAIL_ADDRESS', - "URL", - # 'IBAN_CODE', - # 'AU_TFN', - # 'LOCATION', - # 'AU_MEDICARE', - # 'US_SSN', - # 'MEDICAL_LICENSE' -] -SCRUB_KEYS_HTML = [ - "text", - "canonical_text", - "title", - "state", - "task_description", - "key_char", - "canonical_key_char", - "key_vk", - "children", -] +def filter_log_messages(data): + """ + This function filters log messages by ignoring any message that contains a specific string. + + Args: + data: The input parameter "data" is expected to be data from a loguru logger. + + Returns: + a boolean value indicating whether the message in the input data should be ignored or not. If the + message contains any of the messages in the `messages_to_ignore` list, the function returns `False` + indicating that the message should be ignored. Otherwise, it returns `True` indicating that the + message should not be ignored. + """ + # TODO: ultimately, we want to fix the underlying issues, but for now, we can ignore these messages + messages_to_ignore = [ + "Cannot pickle Objective-C objects", + ] + return not any(msg in data["message"] for msg in messages_to_ignore) diff --git a/openadapt/models.py b/openadapt/models.py index 5248d371f..e76197695 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -269,6 +269,18 @@ def take_screenshot(cls): sct_img = utils.take_screenshot() screenshot = Screenshot(sct_img=sct_img) return screenshot + + def crop_active_window(self, action_event): + window_event = action_event.window_event + width_ratio, height_ratio = utils.get_scale_ratios(action_event) + + x0 = window_event.left * width_ratio + y0 = window_event.top * height_ratio + x1 = x0 + window_event.width * width_ratio + y1 = y0 + window_event.height * height_ratio + + box = (x0, y0, x1, y1) + self._image = self._image.crop(box) class WindowEvent(db.Base): diff --git a/openadapt/record.py b/openadapt/record.py index b406f5bdb..04872ac1e 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -22,7 +22,7 @@ import fire import mss.tools -from openadapt import config, crud, scrub, utils, window +from openadapt import config, crud, utils, window EVENT_TYPES = ("screen", "action", "window") @@ -242,7 +242,6 @@ def on_move( y: int, injected: bool, ) -> None: - logger.debug(f"{x=} {y=} {injected=}") if not injected: trigger_action_event( @@ -251,7 +250,7 @@ def on_move( "name": "move", "mouse_x": x, "mouse_y": y, - } + }, ) @@ -273,7 +272,7 @@ def on_click( "mouse_y": y, "mouse_button_name": button.name, "mouse_pressed": pressed, - } + }, ) @@ -295,7 +294,7 @@ def on_scroll( "mouse_y": y, "mouse_dx": dx, "mouse_dy": dy, - } + }, ) @@ -311,8 +310,7 @@ def handle_key( "vk", ] attrs = { - f"key_{attr_name}": getattr(key, attr_name, None) - for attr_name in attr_names + f"key_{attr_name}": getattr(key, attr_name, None) for attr_name in attr_names } logger.debug(f"{attrs=}") canonical_attrs = { @@ -320,14 +318,7 @@ def handle_key( for attr_name in attr_names } logger.debug(f"{canonical_attrs=}") - trigger_action_event( - event_q, - { - "name": event_name, - **attrs, - **canonical_attrs - } - ) + trigger_action_event(event_q, {"name": event_name, **attrs, **canonical_attrs}) def read_screen_events( @@ -358,8 +349,8 @@ def read_screen_events( def read_window_events( event_q: queue.Queue, - terminate_event: multiprocessing.Event, - recording_timestamp: float, + terminate_event: multiprocessing.Event, + recording_timestamp: float, ) -> None: """ Read window events and add them to the event queue. @@ -378,10 +369,9 @@ def read_window_events( window_data = window.get_active_window_data() if not window_data: continue - if ( - window_data["title"] != prev_window_data.get("title") or - window_data["window_id"] != prev_window_data.get("window_id") - ): + if window_data["title"] != prev_window_data.get("title") or window_data[ + "window_id" + ] != prev_window_data.get("window_id"): # TODO: fix exception sometimes triggered by the next line on win32: # File "\Python39\lib\threading.py" line 917, in run # File "...\openadapt\record.py", line 277, in read window events @@ -394,15 +384,17 @@ def read_window_events( logger.info(f"{_window_data=}") if window_data != prev_window_data: logger.debug("queuing window event for writing") - event_q.put(Event( - utils.get_timestamp(), - "window", - window_data, - )) + event_q.put( + Event( + utils.get_timestamp(), + "window", + window_data, + ) + ) prev_window_data = window_data -def performance_stats_writer ( +def performance_stats_writer( perf_q: multiprocessing.Queue, recording_timestamp: float, terminate_event: multiprocessing.Event, @@ -428,7 +420,10 @@ def performance_stats_writer ( continue crud.insert_perf_stat( - recording_timestamp, event_type, start_time, end_time, + recording_timestamp, + event_type, + start_time, + end_time, ) logger.info("performance stats writer done") @@ -468,25 +463,21 @@ def create_recording( def read_keyboard_events( event_q: queue.Queue, - terminate_event: multiprocessing.Event, - recording_timestamp: float, + terminate_event: multiprocessing.Event, + recording_timestamp: float, ) -> None: - - def on_press(event_q, key, injected): canonical_key = keyboard_listener.canonical(key) logger.debug(f"{key=} {injected=} {canonical_key=}") if not injected: handle_key(event_q, "press", key, canonical_key) - def on_release(event_q, key, injected): canonical_key = keyboard_listener.canonical(key) logger.debug(f"{key=} {injected=} {canonical_key=}") if not injected: handle_key(event_q, "release", key, canonical_key) - utils.set_start_time(recording_timestamp) keyboard_listener = keyboard.Listener( on_press=partial(on_press, event_q), @@ -524,7 +515,7 @@ def record( """ utils.configure_logging(logger, LOG_LEVEL) - logger.info(f"{scrub.scrub_text(task_description)=}") + logger.info(f"{task_description=}") recording = create_recording(task_description) recording_timestamp = recording.timestamp @@ -650,5 +641,6 @@ def record( logger.info(f"saved {recording_timestamp=}") + if __name__ == "__main__": fire.Fire(record) diff --git a/openadapt/scrub.py b/openadapt/scrub.py index cfed2ca27..0c2d820ae 100644 --- a/openadapt/scrub.py +++ b/openadapt/scrub.py @@ -24,13 +24,9 @@ from openadapt import config, utils -SCRUB_PROVIDER_TRF = NlpEngineProvider( - nlp_configuration=config.SCRUB_CONFIG_TRF -) +SCRUB_PROVIDER_TRF = NlpEngineProvider(nlp_configuration=config.SCRUB_CONFIG_TRF) NLP_ENGINE_TRF = SCRUB_PROVIDER_TRF.create_engine() -ANALYZER_TRF = AnalyzerEngine( - nlp_engine=NLP_ENGINE_TRF, supported_languages=["en"] -) +ANALYZER_TRF = AnalyzerEngine(nlp_engine=NLP_ENGINE_TRF, supported_languages=["en"]) ANONYMIZER = AnonymizerEngine() IMAGE_REDACTOR = ImageRedactorEngine(ImageAnalyzerEngine(ANALYZER_TRF)) SCRUBBING_ENTITIES = [ @@ -50,10 +46,6 @@ def scrub_text(text: str, is_separated: bool = False) -> str: Returns: str: Scrubbed text """ - - if config.SCRUB_ENABLED is False: - return text - if text is None: return None @@ -90,9 +82,7 @@ def scrub_text(text: str, is_separated: bool = False) -> str: text.startswith(config.ACTION_TEXT_NAME_PREFIX) or text.endswith(config.ACTION_TEXT_NAME_SUFFIX) ): - anonymized_results.text = config.ACTION_TEXT_SEP.join( - anonymized_results.text - ) + anonymized_results.text = config.ACTION_TEXT_SEP.join(anonymized_results.text) return anonymized_results.text @@ -108,14 +98,11 @@ def scrub_text_all(text: str) -> str: str: Scrubbed text """ - if config.SCRUB_ENABLED is False: - return text - return config.SCRUB_CHAR * len(text) def scrub_image( - image: Image, fill_color=config.DEFAULT_SCRUB_FILL_COLOR + image: Image, fill_color=config.SCRUB_FILL_COLOR ) -> Image: """ Scrub the image of all PII/PHI using Presidio Image Redactor @@ -126,10 +113,6 @@ def scrub_image( Returns: PIL.Image: The scrubbed image with PII and PHI removed. """ - - if config.SCRUB_ENABLED is False: - return image - redacted_image = IMAGE_REDACTOR.redact( image, fill=fill_color, entities=SCRUBBING_ENTITIES ) @@ -189,10 +172,6 @@ def _scrub_text_item( Returns: str: The scrubbed value """ - - if config.SCRUB_ENABLED is False: - return value - if key in ("text", "canonical_text"): return scrub_text(value, is_separated=True) if force_scrub_children: @@ -200,9 +179,7 @@ def _scrub_text_item( return scrub_text(value) -def _should_scrub_list_item( - item: Any, key: Any, list_keys: List[str] -) -> bool: +def _should_scrub_list_item(item: Any, key: Any, list_keys: List[str]) -> bool: """ Check if the key and item should be scrubbed and are of correct instance. @@ -239,10 +216,6 @@ def _scrub_list_item( Returns: dict/str: The scrubbed dict/value respectively """ - - if config.SCRUB_ENABLED is False: - return item - if isinstance(item, dict): return scrub_dict( item, list_keys, force_scrub_children=force_scrub_children @@ -265,10 +238,6 @@ def scrub_dict( Returns: dict: The scrubbed dict with PII and PHI removed. """ - - if config.SCRUB_ENABLED is False: - return input_dict - if list_keys is None: list_keys = config.SCRUB_KEYS_HTML @@ -285,9 +254,7 @@ def scrub_dict( scrubbed_dict[key] = scrubbed_text elif isinstance(value, list): scrubbed_list = [ - _scrub_list_item( - item, key, list_keys, force_scrub_children - ) + _scrub_list_item(item, key, list_keys, force_scrub_children) if _should_scrub_list_item(item, key, list_keys) else item for item in value @@ -296,9 +263,7 @@ def scrub_dict( force_scrub_children = False elif isinstance(value, dict): if isinstance(key, str) and key == "state": - scrubbed_dict[key] = scrub_dict( - value, list_keys, scrub_all=True - ) + scrubbed_dict[key] = scrub_dict(value, list_keys, scrub_all=True) else: scrubbed_dict[key] = scrub_dict(value, list_keys) else: @@ -307,9 +272,7 @@ def scrub_dict( return scrubbed_dict -def scrub_list_dicts( - input_list: List[Dict], list_keys: List = None -) -> List[Dict]: +def scrub_list_dicts(input_list: List[Dict], list_keys: List = None) -> List[Dict]: """ Scrub the list of dicts of all PII/PHI using Presidio ANALYZER.TRF and Anonymizer. @@ -320,10 +283,6 @@ def scrub_list_dicts( Returns: list[dict]: The scrubbed list of dicts with PII and PHI removed. """ - - if config.SCRUB_ENABLED is False: - return input_list - scrubbed_list_dicts = [] for input_dict in input_list: scrubbed_list_dicts.append(scrub_dict(input_dict, list_keys)) diff --git a/openadapt/utils.py b/openadapt/utils.py index 0abcf00df..9c0b08cde 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -15,7 +15,7 @@ import mss.base import numpy as np -from openadapt import common, config, scrub +from openadapt import common, config EMPTY = (None, [], {}, "") @@ -26,16 +26,24 @@ def configure_logging(logger, log_level): log_level_override = os.getenv("LOG_LEVEL") log_level = log_level_override or log_level logger.remove() - logger.add(sys.stderr, level=log_level) + logger.add( + sys.stderr, + level=log_level, + filter=config.filter_log_messages if config.IGNORE_WARNINGS else None, + ) logger.debug(f"{log_level=}") def row2dict(row, follow=True): if isinstance(row, dict): return row - try_follow = [ - "children", - ] if follow else [] + try_follow = ( + [ + "children", + ] + if follow + else [] + ) to_follow = [key for key in try_follow if hasattr(row, key)] # follow children recursively @@ -94,16 +102,15 @@ def rows2dicts( for row_dict in row_dicts: for key in list(row_dict.keys()): value = row_dict[key] - if ( - len(key_to_values[key]) <= 1 - or drop_empty and value in EMPTY - ): + if len(key_to_values[key]) <= 1 or drop_empty and value in EMPTY: del row_dict[key] for row_dict in row_dicts: # TODO: keep attributes in children which vary across parents if "children" in row_dict: row_dict["children"] = rows2dicts( - row_dict["children"], drop_empty, drop_constant, + row_dict["children"], + drop_empty, + drop_constant, ) return row_dicts @@ -117,10 +124,12 @@ def get_double_click_interval_seconds(): return get_double_click_interval_seconds.override_value if sys.platform == "darwin": from AppKit import NSEvent + return NSEvent.doubleClickInterval() elif sys.platform == "win32": # https://stackoverflow.com/a/31686041/95989 from ctypes import windll + return windll.user32.GetDoubleClickTime() / 1000 else: raise Exception(f"Unsupported {sys.platform=}") @@ -134,10 +143,12 @@ def get_double_click_distance_pixels(): # TODO: do this more robustly; see: # https://forum.xojo.com/t/get-allowed-unit-distance-between-doubleclicks-on-macos/35014/7 from AppKit import NSPressGestureRecognizer + return NSPressGestureRecognizer.new().allowableMovement() elif sys.platform == "win32": import win32api import win32con + x = win32api.GetSystemMetrics(win32con.SM_CXDOUBLECLK) y = win32api.GetSystemMetrics(win32con.SM_CYDOUBLECLK) if x != y: @@ -155,12 +166,15 @@ def get_monitor_dims(): return monitor_width, monitor_height +# TODO: move parameters to config def draw_ellipse( - x, y, image, - width_pct=.03, - height_pct=.03, - fill_transparency=.25, - outline_transparency=.5, + x, + y, + image, + width_pct=0.03, + height_pct=0.03, + fill_transparency=0.25, + outline_transparency=0.5, outline_width=2, ): overlay = Image.new("RGBA", image.size) @@ -197,7 +211,10 @@ def get_font(original_font_name, font_size): def draw_text( - x, y, text, image, + x, + y, + text, + image, font_size_pct=0.01, font_name="Arial.ttf", fill=(255, 0, 0), @@ -240,7 +257,11 @@ def draw_text( def draw_rectangle( - x0, y0, x1, y1, image, + x0, + y0, + x1, + y1, + image, bg_color=(0, 0, 0), fg_color=(255, 255, 255), outline_color=(255, 0, 0), @@ -276,10 +297,10 @@ def get_scale_ratios(action_event): def display_event( action_event, - marker_width_pct=.03, - marker_height_pct=.03, - marker_fill_transparency=.25, - marker_outline_transparency=.5, + marker_width_pct=0.03, + marker_height_pct=0.03, + marker_fill_transparency=0.25, + marker_outline_transparency=0.5, diff=False, ): recording = action_event.recording @@ -304,11 +325,15 @@ def display_event( if diff_bbox: x0, y0, x1, y1 = diff_bbox image = draw_rectangle( - x0, y0, x1, y1, image, + x0, + y0, + x1, + y1, + image, outline_color=(255, 0, 0), bg_transparency=0, fg_transparency=0, - #outline_transparency=.75, + # outline_transparency=.75, outline_width=20, ) @@ -328,7 +353,8 @@ def display_event( x = recording.monitor_width * width_ratio / 2 y = recording.monitor_height * height_ratio / 2 text = action_event.text - text = scrub.scrub_text(text, is_separated=True) + if config.SCRUB_ENABLED: + text = __import__("openadapt").scrub.scrub_text(text, is_separated=True) image = draw_text(x, y, text, image, outline=True) else: raise Exception("unhandled {action_event.name=}") @@ -381,11 +407,9 @@ def take_screenshot() -> mss.base.ScreenShot: def get_strategy_class_by_name(): from openadapt.strategies import BaseReplayStrategy + strategy_classes = BaseReplayStrategy.__subclasses__() - class_by_name = { - cls.__name__: cls - for cls in strategy_classes - } + class_by_name = {cls.__name__: cls for cls in strategy_classes} logger.debug(f"{class_by_name=}") return class_by_name @@ -408,6 +432,7 @@ def plot_performance(recording_timestamp: float = None) -> None: if not recording_timestamp: # avoid circular import from openadapt import crud + recording_timestamp = crud.get_latest_recording().timestamp perf_stats = crud.get_perf_stats(recording_timestamp) perf_stat_dicts = rows2dicts(perf_stats) @@ -416,19 +441,13 @@ def plot_performance(recording_timestamp: float = None) -> None: perf_stat["event_type"], perf_stat["start_time"] ) start_time_delta = perf_stat["start_time"] - prev_start_time - type_to_start_time_deltas[perf_stat["event_type"]].append( - start_time_delta - ) - type_to_prev_start_time[perf_stat["event_type"]] = ( - perf_stat["start_time"] - ) + type_to_start_time_deltas[perf_stat["event_type"]].append(start_time_delta) + type_to_prev_start_time[perf_stat["event_type"]] = perf_stat["start_time"] type_to_proc_times[perf_stat["event_type"]].append( perf_stat["end_time"] - perf_stat["start_time"] ) type_to_count[perf_stat["event_type"]] += 1 - type_to_timestamps[perf_stat["event_type"]].append( - perf_stat["start_time"] - ) + type_to_timestamps[perf_stat["event_type"]].append(perf_stat["start_time"]) y_data = {"proc_times": {}, "start_time_deltas": {}} for i, event_type in enumerate(type_to_count): @@ -438,7 +457,7 @@ def plot_performance(recording_timestamp: float = None) -> None: y_data["proc_times"][event_type] = proc_times y_data["start_time_deltas"][event_type] = start_time_deltas - fig, axes = plt.subplots(2, 1, sharex=True, figsize=(20,10)) + fig, axes = plt.subplots(2, 1, sharex=True, figsize=(20, 10)) for i, data_type in enumerate(y_data): for event_type in y_data[data_type]: x = type_to_timestamps[event_type] diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 6511522fd..7fec235ae 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -24,15 +24,20 @@ row2dict, rows2dicts, ) -from openadapt import scrub +from openadapt import config + +SCRUB = config.SCRUB_ENABLED +if SCRUB: + from openadapt import scrub LOG_LEVEL = "INFO" MAX_EVENTS = None MAX_TABLE_CHILDREN = 5 PROCESS_EVENTS = True IMG_WIDTH_PCT = 60 -CSS = string.Template(""" +CSS = string.Template( + """ table { outline: 1px solid black; } @@ -66,7 +71,8 @@ .screenshot:active img:nth-child(3) { display: block; } -""").substitute( +""" +).substitute( IMG_WIDTH_PCT=IMG_WIDTH_PCT, ) @@ -117,16 +123,18 @@ def dict2html(obj, max_children=MAX_TABLE_CHILDREN): children = indicate_missing(children, all_children, "...") html_str = "\n".join(children) elif isinstance(obj, dict): - rows_html = "\n".join([ - f""" + rows_html = "\n".join( + [ + f""" {format_key(key, value)} {dict2html(value, max_children)} """ - for key, value in obj.items() - if value not in EMPTY - ]) + for key, value in obj.items() + if value not in EMPTY + ] + ) html_str = f"{rows_html}
" else: html_str = html.escape(str(obj)) @@ -137,16 +145,22 @@ def main(): configure_logging(logger, LOG_LEVEL) recording = get_latest_recording() - recording.task_description = scrub.scrub_text(recording.task_description) + if SCRUB: + scrub.scrub_text(recording.task_description) logger.debug(f"{recording=}") meta = {} action_events = get_events(recording, process=PROCESS_EVENTS, meta=meta) event_dicts = rows2dicts(action_events) - event_dicts = scrub.scrub_list_dicts(event_dicts) + if SCRUB: + event_dicts = scrub.scrub_list_dicts(event_dicts) logger.info(f"event_dicts=\n{pformat(event_dicts)}") + recording_dict = row2dict(recording) + if SCRUB: + recording_dict = scrub.scrub_dict(recording_dict) + rows = [ row( Div( @@ -155,7 +169,7 @@ def main(): ), row( Div( - text=f"{dict2html(scrub.scrub_dict(row2dict(recording)))}", + text=f"{dict2html(recording_dict)}", ), ), row( @@ -163,7 +177,7 @@ def main(): text=f"{dict2html(meta)}", width_policy="max", ), - ) + ), ] logger.info(f"{len(action_events)=}") for idx, action_event in enumerate(action_events): @@ -172,17 +186,29 @@ def main(): image = display_event(action_event) diff = display_event(action_event, diff=True) mask = action_event.screenshot.diff_mask - image = scrub.scrub_image(image) - diff = scrub.scrub_image(diff) - mask = scrub.scrub_image(mask) + + if SCRUB: + image = scrub.scrub_image(image) + diff = scrub.scrub_image(diff) + mask = scrub.scrub_image(mask) + image_utf8 = image2utf8(image) diff_utf8 = image2utf8(diff) mask_utf8 = image2utf8(mask) width, height = image.size - rows.append([ - row( - Div( - text=f""" + + action_event_dict = row2dict(action_event) + window_event_dict = row2dict(action_event.window_event) + + if SCRUB: + action_event_dict = scrub.scrub_dict(action_event_dict) + window_event_dict = scrub.scrub_dict(window_event_dict) + + rows.append( + [ + row( + Div( + text=f"""
- {dict2html(scrub.scrub_dict(row2dict(action_event.window_event)), None)} + {dict2html(window_event_dict , None)}
""", - ), - Div( - text=f""" + ), + Div( + text=f""" - {dict2html(scrub.scrub_dict(row2dict(action_event)))} + {dict2html(action_event_dict)}
""" + ), ), - ), - ]) + ] + ) title = f"recording-{recording.id}" fname_out = f"recording-{recording.id}.html" @@ -229,13 +256,11 @@ def main(): ) ) - def cleanup(): os.remove(fname_out) removed = not os.path.exists(fname_out) logger.info(f"{removed=}") - Timer(1, cleanup).start() diff --git a/tests/openadapt/test_crop.py b/tests/openadapt/test_crop.py new file mode 100644 index 000000000..98ef7acb3 --- /dev/null +++ b/tests/openadapt/test_crop.py @@ -0,0 +1,35 @@ +from unittest.mock import Mock +from unittest import mock +from PIL import Image +import numpy as np +from openadapt.models import Screenshot + +def test_crop_active_window(): + # Create a mock action event with a mock window event + action_event_mock = Mock() + window_event_mock = Mock() + + # Define window_event's attributes + window_event_mock.left = 100 + window_event_mock.top = 100 + window_event_mock.width = 300 + window_event_mock.height = 300 + action_event_mock.window_event = window_event_mock + + # Mock the utils.get_scale_ratios to return some fixed ratios + with mock.patch('openadapt.utils.get_scale_ratios', return_value=(1, 1)): + # Create a dummy image and put it in a Screenshot object + image = Image.new('RGB', (500, 500), color='white') + screenshot = Screenshot() + screenshot._image = image + + # Store original image size for comparison + original_size = screenshot._image.size + + # Perform the cropping operation + screenshot.crop_active_window(action_event=action_event_mock) + + # Verify that the image size has been reduced + assert ((screenshot._image.size[0] < original_size[0]) or + (screenshot._image.size[1] < original_size[1])) + diff --git a/tests/openadapt/test_scrub.py b/tests/openadapt/test_scrub.py index 87549ede4..ab607d10b 100644 --- a/tests/openadapt/test_scrub.py +++ b/tests/openadapt/test_scrub.py @@ -37,9 +37,7 @@ def test_scrub_image() -> None: # Count the number of pixels having the color of the mask mask_pixels = sum( - 1 - for pixel in scrubbed_image.getdata() - if pixel == config.DEFAULT_SCRUB_FILL_COLOR + 1 for pixel in scrubbed_image.getdata() if pixel == config.SCRUB_FILL_COLOR ) total_pixels = scrubbed_image.width * scrubbed_image.height @@ -104,9 +102,7 @@ def test_scrub_credit_card() -> None: """ assert ( - scrub.scrub_text( - "My credit card number is 4234-5678-9012-3456 and " - ) + scrub.scrub_text("My credit card number is 4234-5678-9012-3456 and ") ) == "My credit card number is ******************* and " @@ -219,5 +215,6 @@ def test_scrub_all_together() -> None: " He was born on 01/01/1980." ) + if __name__ == "__main__": test_scrub_image()