From a5ecebf29708ca7d2dee861240b91eb2fe3eaf1d Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 10 Jul 2024 10:44:48 +0200 Subject: [PATCH] Improved setup instructions, added notebook trusting to container --- Dockerfile | 6 +- README.md | 122 ++++++++++++++++++++++++++-------------- docker_build_and_run.sh | 2 +- 3 files changed, 85 insertions(+), 45 deletions(-) diff --git a/Dockerfile b/Dockerfile index c4fd0d0..1d3b2b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM python:3.11-slim # Set environment variables to make Python print directly to the terminal and avoid .pyc files. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 # Install system dependencies required for the project. RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -52,4 +52,4 @@ RUN poetry install --with dev # Entrypoint should be a shell in the workdir with poetry shell activated # Before that, the project should be installed with poetry install -ENTRYPOINT ["/bin/bash", "-c", "poetry install --with dev && poetry shell && $0 $@"] +ENTRYPOINT ["/bin/bash", "-c", "poetry install --with dev && poetry run jupyter trust notebooks/*.ipynb docs/02_notebooks/*.ipynb && poetry shell && $0 $@"] diff --git a/README.md b/README.md index 19b1ad5..1190680 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # Generative AI Capabilities for Penpot -This project explores applications of generative AI in the context of [Penpot](https://penpot.app), +This project explores applications of generative AI in the context +of [Penpot](https://penpot.app), an open-source design and prototyping platform. -While some of the capabilities are specific to Penpot, the project also aims to provide general -functionality for the generation and augmentation of design assets provided in standard formats +While some of the capabilities are specific to Penpot, the project also aims to +provide general +functionality for the generation and augmentation of design assets provided in +standard formats such as SVG. @@ -13,85 +16,104 @@ such as SVG. - [Use Cases](#use-cases) - * [Generating Variations of Vector Shapes](#generating-variations-of-vector-shapes) - * [Variation Style Transfer](#variation-style-transfer) - * [Naming Shapes Semantically](#naming-shapes-semantically) - * [Hierarchy Inference](#hierarchy-inference) + * [Generating Variations of Vector Shapes](#generating-variations-of-vector-shapes) + * [Variation Style Transfer](#variation-style-transfer) + * [Naming Shapes Semantically](#naming-shapes-semantically) + * [Hierarchy Inference](#hierarchy-inference) - [Development Environment](#development-environment) - * [Python Virtual Environment](#python-virtual-environment) - * [Docker Setup](#docker-setup) - * [Codespaces](#codespaces) - * [Secrets, Configuration and Credentials](#secrets-configuration-and-credentials) + * [Python Virtual Environment](#python-virtual-environment) + * [Docker Setup](#docker-setup) + * [Codespaces](#codespaces) + * [Secrets, Configuration and Credentials](#secrets-configuration-and-credentials) ## Use Cases -Our use cases build upon state-of-the-art large language models/vision language models. -Particularly Anthropic's *Claude 3.5 Sonnet* model proved to have a solid understanding of vector +Our use cases build upon state-of-the-art large language models/vision language +models. +Particularly Anthropic's *Claude 3.5 Sonnet* model proved to have a solid +understanding of vector graphics and is therefore the model of choice for most our use cases. ### Generating Variations of Vector Shapes -As an inspirational starting point, we consider the problem of generating variations of a given shape. +As an inspirational starting point, we consider the problem of generating +variations of a given shape. ![shape variations](resources/images/use_case_variations.png) -In order to facilitate the generation of variations, it can be very helpful to refactor the original -SVG representation in order to make shapes and cutouts more explicit, avoiding less explicit SVG path +In order to facilitate the generation of variations, it can be very helpful to +refactor the original +SVG representation in order to make shapes and cutouts more explicit, avoiding +less explicit SVG path representations whenever possible. -Furthermore, we typically want to limit the scope of what is varied. For instance, we might want to +Furthermore, we typically want to limit the scope of what is varied. For +instance, we might want to vary only foreground colours or certain inner shapes. -Furthermore, we typically have constraints that shall guide the generation process, e.g. to ensure -that the generated shapes remain close to the original shape, maintaining the semantics, or to ensure +Furthermore, we typically have constraints that shall guide the generation +process, e.g. to ensure +that the generated shapes remain close to the original shape, maintaining the +semantics, or to ensure that colour variations respect the colour palette of the design project. -* Batch job: - * Apply variation generation to registered shapes: [scripts/batch_svg_variations.py](scripts/batch_svg_variations.py) - * Inspection of results: [scripts/web_server.py](scripts/web_server.py) +* Batch job: + * Apply variation generation to registered + shapes: [scripts/batch_svg_variations.py](scripts/batch_svg_variations.py) + * Inspection of results: [scripts/web_server.py](scripts/web_server.py) * Notebooks: - * Variation generation for an icon/logo: [notebooks/svg_variations_icon.ipynb](notebooks/svg_variations_icon.ipynb) - * Variation generation for a UI element: [notebooks/svg_variations_ui_widget.ipynb](notebooks/svg_variations_ui_widget.ipynb) + * Variation generation for an + icon/logo: [notebooks/svg_variations_icon.ipynb](notebooks/svg_variations_icon.ipynb) + * Variation generation for a UI + element: [notebooks/svg_variations_ui_widget.ipynb](notebooks/svg_variations_ui_widget.ipynb) ### Variation Style Transfer -In user interface design, the same principles are often applied to a wide variety of -UI elements and other shapes, e.g. in order to indicate different UI states such as +In user interface design, the same principles are often applied to a wide +variety of +UI elements and other shapes, e.g. in order to indicate different UI states such +as 'hover', 'focus', or 'disabled'. -Given a template indicating how a given UI element is transformed, the task is to apply +Given a template indicating how a given UI element is transformed, the task is +to apply the same transformation to a different shape: ![shape variation style transfer](resources/images/use_case_variation_transfer.png) -Here are some results we obtained with Claude 3.5, showing the presented template +Here are some results we obtained with Claude 3.5, showing the presented +template and subsequently the applied transformation: ![shape variation style transfer/results](resources/videos/use_case_variation_transfer.gif) -* Batch job: - * Apply variation transfer to registered shapes: [scripts/batch_variation_transfer.py](scripts/batch_svg_variation_transfer.py) - * Inspection of results: [scripts/web_server.py](scripts/web_server.py) +* Batch job: + * Apply variation transfer to registered + shapes: [scripts/batch_variation_transfer.py](scripts/batch_svg_variation_transfer.py) + * Inspection of results: [scripts/web_server.py](scripts/web_server.py) * Notebooks: - * Variation transfer for UI elements: [notebooks/svg_variation_transfer_ui_widget.ipynb](notebooks/svg_variation_transfer_ui_widget.ipynb) + * Variation transfer for UI + elements: [notebooks/svg_variation_transfer_ui_widget.ipynb](notebooks/svg_variation_transfer_ui_widget.ipynb) ### Naming Shapes Semantically -Associating shapes with meaningful names can be essential for discoverability, especially +Associating shapes with meaningful names can be essential for discoverability, +especially in large design projects. -We thus consider the problem of finding meaningful names for an existing hierarchy of shapes: +We thus consider the problem of finding meaningful names for an existing +hierarchy of shapes: ![shape naming](resources/images/use_case_shape_naming.png) ### Hierarchy Inference In larger projects, shapes may be grouped suboptimally. -We consider the problem of jointly inferring an updated hierarchy and naming the shapes contained: +We consider the problem of jointly inferring an updated hierarchy and naming the +shapes contained: ![hierarchy inference](resources/images/use_case_hierarchy_inference.png) - ## Development Environment Clone the repository and run @@ -110,6 +132,9 @@ Create a Python 3.11 environment and install the dependencies with poetry install --with dev ``` +You might have to run `jupyter trust ...` for some notebooks to be able to +display SVG content. + ### Docker Setup Build the docker image with @@ -121,21 +146,36 @@ docker build -t penai . and run it with the repository mounted as a volume: ```shell -docker run -it --rm -v "$(pwd)":/workspaces/penai penai +docker run -it --shm-size=1g --rm -v "$(pwd)":/workspaces/penai penai ``` +(The `shm-size` option is necessary for google-webdriver to have enough space +for +rendering things.) + You can also just run `bash docker_build_and_run.sh`, which will do both things for you. -To make a quick check if everything is working , you can run the `pytest` -command from the container. +To make a quick check if everything is working, you can run the `pytest` +command from the container. If you want to run jupyter in the container, you +should start the container with port forwarding +(e.g. adding `-p 8888:8888` to the `docker run` command) +and then start jupyter e.g. with + +```shell +jupyter notebook --ip 0.0.0.0 --port 8888 --allow-root --no-browser +``` + +(the `--ip` option is needed to prevent jupyter from only listening to localhost +and the `--allow-root` option is because the container user is root.) -Note: When using the Windows subsystem for Linux (WSL), you might need to adjust the path for the +Note: When using the Windows subsystem for Linux (WSL), you might need to adjust +the path for the volume. ### Codespaces -The fastest way to get running without any installation is to use GitHub +The fastest way to get running without any installation is to use GitHub Codespaces. The repository has been set up to provide a fully functioning Codespace with everything installed out of the box. You can either paste your `config_local.json` file there or pass the secrets as env vars diff --git a/docker_build_and_run.sh b/docker_build_and_run.sh index ba90f45..e14da13 100755 --- a/docker_build_and_run.sh +++ b/docker_build_and_run.sh @@ -2,4 +2,4 @@ docker build -t penai . -docker run -it --rm -v "$(pwd)":/workspaces/penai penai +docker run -it --rm --shm-size=1g -v "$(pwd)":/workspaces/penai penai