Merge pull request #2029 from fractal-analytics-platform/2018-cleanup…

…-task-activity-templates Cleanup task-activity templates
fractal-analytics-platform · Nov 11, 2024 · 4d8d79e · 4d8d79e
2 parents 4a47890 + db07dec
commit 4d8d79e
Show file tree

Hide file tree

Showing 11 changed files with 172 additions and 98 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,7 @@
     * Update filename and path for task-collection scripts (\#2008).
     * Copy wheel file into `task_group.path` and update `task_group.wheel_path`, for local task collection (\#2020).
     * Set `TaskGroupActivityV2.timestamp_ended` when collections terminate (\#2026).
+    * Refactor bash templates and add `install_from_freeze.sh` (\#2029).
 * SSH internals:
     * Add `FractalSSH.remote_exists` method (\#2008).
 

diff --git a/fractal_server/tasks/v2/local/collect.py b/fractal_server/tasks/v2/local/collect.py
@@ -4,6 +4,7 @@
 import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import Optional
 
 from ..utils_database import create_db_tasks_and_update_task_group
 from fractal_server.app.db import get_sync_db
@@ -26,7 +27,9 @@
 )
 from fractal_server.tasks.v2.utils_templates import customize_template
 from fractal_server.tasks.v2.utils_templates import get_collection_replacements
-from fractal_server.tasks.v2.utils_templates import parse_script_5_stdout
+from fractal_server.tasks.v2.utils_templates import (
+    parse_script_pip_show_stdout,
+)
 from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
 from fractal_server.utils import execute_command_sync
 from fractal_server.utils import get_timestamp
@@ -38,7 +41,7 @@ def _customize_and_run_template(
     template_filename: str,
     replacements: list[tuple[str, str]],
     script_dir: str,
-    prefix: int,
+    prefix: Optional[int] = None,
 ) -> str:
     """
     Customize one of the template bash scripts.
@@ -57,26 +60,25 @@ def _customize_and_run_template(
         raise ValueError(
             f"Invalid {template_filename=} (it must end with '.sh')."
         )
-    template_filename_stripped = template_filename[:-3]
 
-    script_filename = f"{prefix}{template_filename_stripped}"
-    script_path_local = Path(script_dir) / script_filename
+    template_filename_stripped = template_filename
 
+    if prefix is not None:
+        script_filename = f"{prefix}{template_filename_stripped}"
+    else:
+        script_filename = template_filename_stripped
+    script_path_local = Path(script_dir) / script_filename
     # Read template
     customize_template(
         template_name=template_filename,
         replacements=replacements,
         script_path=script_path_local,
     )
-
     cmd = f"bash {script_path_local}"
     logger.debug(f"Now run '{cmd}' ")
-
     stdout = execute_command_sync(command=cmd, logger_name=LOGGER_NAME)
-
     logger.debug(f"Standard output of '{cmd}':\n{stdout}")
     logger.debug(f"_customize_and_run_template {template_filename} - END")
-
     return stdout
 
 
@@ -194,45 +196,37 @@ def collect_package_local(
 
                 # Run script 1
                 stdout = _customize_and_run_template(
-                    template_filename="_1_create_venv.sh",
+                    template_filename="1_create_venv.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
                 # Run script 2
                 stdout = _customize_and_run_template(
-                    template_filename="_2_preliminary_pip_operations.sh",
+                    template_filename="2_pip_install.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
                 # Run script 3
-                stdout = _customize_and_run_template(
-                    template_filename="_3_pip_install.sh",
-                    **common_args,
-                )
-                activity.log = get_current_log(log_file_path)
-                activity = add_commit_refresh(obj=activity, db=db)
-
-                # Run script 4
                 pip_freeze_stdout = _customize_and_run_template(
-                    template_filename="_4_pip_freeze.sh",
+                    template_filename="3_pip_freeze.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
-                # Run script 5
+                # Run script 4
                 stdout = _customize_and_run_template(
-                    template_filename="_5_pip_show.sh",
+                    template_filename="4_pip_show.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
-                pkg_attrs = parse_script_5_stdout(stdout)
+                pkg_attrs = parse_script_pip_show_stdout(stdout)
                 for key, value in pkg_attrs.items():
                     logger.debug(f"Parsed from pip-show: {key}={value}")
                 # Check package_name match between pip show and task-group

diff --git a/fractal_server/tasks/v2/ssh/collect.py b/fractal_server/tasks/v2/ssh/collect.py
@@ -24,7 +24,9 @@
 )
 from fractal_server.tasks.v2.utils_templates import customize_template
 from fractal_server.tasks.v2.utils_templates import get_collection_replacements
-from fractal_server.tasks.v2.utils_templates import parse_script_5_stdout
+from fractal_server.tasks.v2.utils_templates import (
+    parse_script_pip_show_stdout,
+)
 from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
 from fractal_server.utils import get_timestamp
 
@@ -242,45 +244,38 @@ def collect_package_ssh(
 
                 # Run script 1
                 stdout = _customize_and_run_template(
-                    template_filename="_1_create_venv.sh",
+                    template_filename="1_create_venv.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
                 # Run script 2
                 stdout = _customize_and_run_template(
-                    template_filename="_2_preliminary_pip_operations.sh",
+                    template_filename="2_pip_install.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
                 # Run script 3
-                stdout = _customize_and_run_template(
-                    template_filename="_3_pip_install.sh",
-                    **common_args,
-                )
-                activity.log = get_current_log(log_file_path)
-                activity = add_commit_refresh(obj=activity, db=db)
-
-                # Run script 4
                 pip_freeze_stdout = _customize_and_run_template(
-                    template_filename="_4_pip_freeze.sh",
+                    template_filename="3_pip_freeze.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
-                # Run script 5
+                # Run script 4
                 stdout = _customize_and_run_template(
-                    template_filename="_5_pip_show.sh",
+                    template_filename="4_pip_show.sh",
                     **common_args,
                 )
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
-                pkg_attrs = parse_script_5_stdout(stdout)
+                pkg_attrs = parse_script_pip_show_stdout(stdout)
+
                 for key, value in pkg_attrs.items():
                     logger.debug(f"parsed from pip-show: {key}={value}")
                 # Check package_name match between pip show and task-group

diff --git a/...rver/tasks/v2/templates/_1_create_venv.sh → ...erver/tasks/v2/templates/1_create_venv.sh b/...rver/tasks/v2/templates/_1_create_venv.sh → ...erver/tasks/v2/templates/1_create_venv.sh
diff --git a/...rver/tasks/v2/templates/_3_pip_install.sh → ...erver/tasks/v2/templates/2_pip_install.sh b/...rver/tasks/v2/templates/_3_pip_install.sh → ...erver/tasks/v2/templates/2_pip_install.sh
@@ -5,16 +5,23 @@ write_log(){
     echo "[collect-task, $TIMESTAMP] $1"
 }
 
-
 # Variables to be filled within fractal-server
 PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
 INSTALL_STRING=__INSTALL_STRING__
 PINNED_PACKAGE_LIST="__PINNED_PACKAGE_LIST__"
+FRACTAL_MAX_PIP_VERSION="__FRACTAL_MAX_PIP_VERSION__"
 
 TIME_START=$(date +%s)
 
 VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
 
+# Upgrade `pip` and install `setuptools`
+write_log "START upgrade pip and install setuptools"
+"$VENVPYTHON" -m pip install --no-cache-dir "pip<=${FRACTAL_MAX_PIP_VERSION}" --upgrade
+"$VENVPYTHON" -m pip install --no-cache-dir setuptools
+write_log "END   upgrade pip and install setuptools"
+echo
+
 # Install package
 write_log "START install ${INSTALL_STRING}"
 "$VENVPYTHON" -m pip install --no-cache-dir "$INSTALL_STRING"

diff --git a/...erver/tasks/v2/templates/_4_pip_freeze.sh → ...server/tasks/v2/templates/3_pip_freeze.sh b/...erver/tasks/v2/templates/_4_pip_freeze.sh → ...server/tasks/v2/templates/3_pip_freeze.sh
@@ -1,12 +1,5 @@
 set -e
 
-write_log(){
-    TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-    echo "[collect-task, $TIMESTAMP] $1"
-}
-
-
-
 # Variables to be filled within fractal-server
 PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
 

diff --git a/..._server/tasks/v2/templates/_5_pip_show.sh → ...l_server/tasks/v2/templates/4_pip_show.sh b/..._server/tasks/v2/templates/_5_pip_show.sh → ...l_server/tasks/v2/templates/4_pip_show.sh
@@ -11,7 +11,6 @@ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
 PACKAGE_NAME=__PACKAGE_NAME__
 
 
-
 TIME_START=$(date +%s)
 
 VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python

diff --git a/fractal_server/tasks/v2/templates/5_pip_install_from_freeze.sh b/fractal_server/tasks/v2/templates/5_pip_install_from_freeze.sh
@@ -0,0 +1,35 @@
+set -e
+
+write_log(){
+    TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    echo "[collect-task, $TIMESTAMP] $1"
+}
+
+# Variables to be filled within fractal-server
+PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
+PIP_FREEZE_FILE=__PIP_FREEZE_FILE__
+FRACTAL_MAX_PIP_VERSION=__FRACTAL_MAX_PIP_VERSION__
+
+TIME_START=$(date +%s)
+
+VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
+
+# Upgrade `pip` and install `setuptools`
+write_log "START upgrade pip and install setuptools"
+"$VENVPYTHON" -m pip install --no-cache-dir "pip<=${FRACTAL_MAX_PIP_VERSION}" --upgrade
+"$VENVPYTHON" -m pip install --no-cache-dir setuptools
+write_log "END   upgrade pip and install setuptools"
+echo
+
+# Install from pip-freeze file
+write_log "START installing requirements from ${PIP_FREEZE_FILE}"
+"$VENVPYTHON" -m pip install -r "${PIP_FREEZE_FILE}"
+write_log "END   installing requirements from ${PIP_FREEZE_FILE}"
+echo
+
+# End
+TIME_END=$(date +%s)
+write_log "All good up to here."
+write_log "Elapsed: $((TIME_END - TIME_START)) seconds"
+write_log "Exit."
+echo
diff --git a/fractal_server/tasks/v2/templates/_2_preliminary_pip_operations.sh b/fractal_server/tasks/v2/templates/_2_preliminary_pip_operations.sh
diff --git a/fractal_server/tasks/v2/utils_templates.py b/fractal_server/tasks/v2/utils_templates.py
@@ -38,9 +38,9 @@ def customize_template(
         f.write(script_data)
 
 
-def parse_script_5_stdout(stdout: str) -> dict[str, str]:
+def parse_script_pip_show_stdout(stdout: str) -> dict[str, str]:
     """
-    Parse standard output of template 5.
+    Parse standard output of 4_pip_show.sh
     """
     searches = [
         ("Python interpreter:", "python_bin"),
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,7 +11,6 @@ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
		PACKAGE_NAME=__PACKAGE_NAME__



		TIME_START=$(date +%s)

		VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
Expand Down