feat: add EDX_PLATFORM_NEW_ASSET_PROCESSING flag

When enabled, this flag tells Tutor's openedx-assets script
(which processes assets using Python and edx-platform's Paver tools)
to instead proxy through to edx-platform's new asset processing system
(`scripts/build-assets.sh` and `./manage.py collectstatic`).

Also, when enabled, openedx-assets will print out the shell command
that could have been used to directly invoke the new asset processing
system. In the future, openedx-assets will be deprecated, and the
direct shell command will be the canonical way of compiling assets.

Part of: openedx/edx-platform#31798
ADR: https://github.com/openedx/edx-platform/blob/master/docs/decisions/0017-reimplement-asset-processing.rst

changelog.d/20230414_161756_kdmc_assets_sh.md

@@ -0,0 +1,12 @@
+<!--
+Create a changelog entry for every new user-facing change. Please respect the following instructions:
+- Indicate breaking changes by prepending an explosion 💥 character.
+- Prefix your changes with either [Bugfix], [Improvement], [Feature], [Security], [Deprecation].
+- You may optionally append "(by @<author>)" at the end of the line, where "<author>" is either one (just one)
+  of your GitHub username, real name or affiliated organization. These affiliations will be displayed in
+  the release notes for every release.
+-->
+
+<!-- - 💥[Feature] Foobarize the blorginator. This breaks plugins by renaming the `FOO_DO` filter to `BAR_DO`. (by @regisb) -->
+<!-- - [Improvement] This is a non-breaking change. Life is good. (by @billgates) -->
+- [Feature] Add	`EDX_PLATFORM_NEW_ASSET_SYSTEM` flag, defaulting off. When enabled, the experimental new edx-platform asset processing system will be used. The new system should produce identical output, but quicker and with better logging. In the future, this will become the default, supporting further image build optimizations. (@kdmccormick)

tutor/templates/build/openedx/Dockerfile

@@ -167,6 +167,11 @@ ENV PATH /openedx/bin:${PATH}
 
 {{ patch("openedx-dockerfile-pre-assets") }}
 
+# For the experimental new asset processing:
+# https://github.com/openedx/edx-platform/issues/31604
+ENV EDX_PLATFORM_NEW_ASSET_PROCESSING="{{ EDX_PLATFORM_NEW_ASSET_PROCESSING }}"
+ENV EDX_PLATFORM_ASSET_OPTS="--static-root /openedx/staticfiles --theme-dirs /openedx/themes"
+
 # Collect production assets. By default, only assets from the default theme
 # will be processed. This makes the docker image lighter and faster to build.
 # Only the custom themes added to /openedx/themes will be compiled.

tutor/templates/build/openedx/bin/openedx-assets

@@ -4,14 +4,18 @@ import argparse
 import os
 import subprocess
 import sys
+import shlex
 import traceback
 
 from path import Path
 
 from pavelib import assets
 
 
+DEFAULT_ENV = "prod"
+DEFAULT_SYSTEMS = ["lms", "cms"]
 DEFAULT_STATIC_ROOT = "/openedx/staticfiles"
+DEFAULT_THEMES = ["all"]
 DEFAULT_THEMES_DIR = "/openedx/themes"
 
 
@@ -25,19 +29,19 @@ def main():
     npm.set_defaults(func=run_npm)
 
     build = subparsers.add_parser("build", help="Build all assets")
-    build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod")
+    build.add_argument("-e", "--env", choices=["prod", "dev"], default=DEFAULT_ENV)
     build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR])
-    build.add_argument("--themes", nargs="+", default=["all"])
+    build.add_argument("--themes", nargs="+", default=DEFAULT_THEMES)
     build.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT)
-    build.add_argument("--systems", nargs="+", default=["lms", "cms"])
+    build.add_argument("--systems", nargs="+", default=DEFAULT_SYSTEMS)
     build.set_defaults(func=run_build)
 
     xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule")
     xmodule.set_defaults(func=run_xmodule)
 
     webpack = subparsers.add_parser("webpack", help="Run webpack")
     webpack.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT)
-    webpack.add_argument("-e", "--env", choices=["prod", "dev"], default="prod")
+    webpack.add_argument("-e", "--env", choices=["prod", "dev"], default=DEFAULT_ENV)
     webpack.set_defaults(func=run_webpack)
 
     common = subparsers.add_parser(
@@ -86,7 +90,72 @@ def main():
     watch_themes.set_defaults(func=run_watch_themes)
 
     args = parser.parse_args()
-    args.func(args)
+    if should_use_new_implementation():
+        new_command = get_new_command(args)
+        print("Running experimental new asset processing implementation!")
+        print("You can also run this directly by using the shell command:")
+        print(shlex.join(get_new_command(args)))
+        subprocess.check_call(new_command)
+    else:
+        args.func(args)
+
+
+default_args = [
+    "--themes-dirs",
+    DEFAULT_THEMES_DIR,
+    "--static-root",
+    DEFAULT_STATIC_ROOT,
+]
+
+
+def should_use_new_implementation():
+    return os.environ.get("EDX_PLATFORM_NEW_ASSET_PROCESSING", "").lower() == "true"
+
+
+def get_new_command(args):
+    if args.func is run_collect:
+        return [
+            "bash",
+            "-c",
+            "./manage.py lms collecstatic --noinput && ./manage.py cms collecstatic --noinput",
+        ]
+    return [
+        "scripts/build-assets.sh",
+        *{
+            run_build: [],
+            run_xmodule: ["xmodule"],
+            run_npm: ["npm"],
+            run_webpack: ["webpack"],
+            run_common: ["css"],
+            run_themes: ["themes"],
+            run_watch_themes: ["--watch"]
+        }[args.func],
+        *(
+            ["--env", args.env]
+            if getattr(args, "env", DEFAULT_ENV) != DEFAULT_ENV
+            else []
+        ),
+        *(
+            ["--static-root", args.static_root]
+            if getattr(args, "static_root", DEFAULT_STATIC_ROOT) != DEFAULT_STATIC_ROOT
+            else []
+        ),
+        *(
+            ["--systems", *args.systems]
+            if set(getattr(args, "systems", DEFAULT_SYSTEMS)) != set(DEFAULT_SYSTEMS)
+            else []
+        ),
+        *(
+            ["--themes", *args.themes]
+            if set(getattr(args, "themes", DEFAULT_THEMES)) != set(DEFAULT_THEMES)
+            else []
+        ),
+        *(
+            ["--theme-dirs", *args.theme_dirs]
+            if getattr(args, "theme_dirs", [DEFAULT_THEMES_DIR]) != [DEFAULT_THEMES_DIR]
+            else []
+        ),
+    ]
 
 
 def run_build(args):
@@ -187,7 +256,6 @@ def list_subdirectories(path):
         if os.path.isdir(os.path.join(path, subpath))
     ]
 
-
 class ThemeWatcher(assets.SassWatcher):
     def __init__(self, theme_dir):
         super(ThemeWatcher, self).__init__()

tutor/templates/config/defaults.yml

@@ -21,6 +21,7 @@ DOCKER_IMAGE_REDIS: "docker.io/redis:6.2.6"
 DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.95-r0-2"
 EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git"
 EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}"
+EDX_PLATFORM_NEW_ASSET_PROCESSING: false
 ELASTICSEARCH_HOST: "elasticsearch"
 ELASTICSEARCH_PORT: 9200
 ELASTICSEARCH_SCHEME: "http"