diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a23c426c..bd7ff31699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * Update filename and path for task-collection scripts (\#2008). * Copy wheel file into `task_group.path` and update `task_group.wheel_path`, for local task collection (\#2020). * Set `TaskGroupActivityV2.timestamp_ended` when collections terminate (\#2026). + * Refactor bash templates and add `install_from_freeze.sh` (\#2029). * SSH internals: * Add `FractalSSH.remote_exists` method (\#2008). diff --git a/fractal_server/tasks/v2/local/collect.py b/fractal_server/tasks/v2/local/collect.py index 3e6ae74fff..d6dac9ae52 100644 --- a/fractal_server/tasks/v2/local/collect.py +++ b/fractal_server/tasks/v2/local/collect.py @@ -4,6 +4,7 @@ import time from pathlib import Path from tempfile import TemporaryDirectory +from typing import Optional from ..utils_database import create_db_tasks_and_update_task_group from fractal_server.app.db import get_sync_db @@ -26,7 +27,9 @@ ) from fractal_server.tasks.v2.utils_templates import customize_template from fractal_server.tasks.v2.utils_templates import get_collection_replacements -from fractal_server.tasks.v2.utils_templates import parse_script_5_stdout +from fractal_server.tasks.v2.utils_templates import ( + parse_script_pip_show_stdout, +) from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER from fractal_server.utils import execute_command_sync from fractal_server.utils import get_timestamp @@ -38,7 +41,7 @@ def _customize_and_run_template( template_filename: str, replacements: list[tuple[str, str]], script_dir: str, - prefix: int, + prefix: Optional[int] = None, ) -> str: """ Customize one of the template bash scripts. @@ -57,26 +60,25 @@ def _customize_and_run_template( raise ValueError( f"Invalid {template_filename=} (it must end with '.sh')." ) - template_filename_stripped = template_filename[:-3] - script_filename = f"{prefix}{template_filename_stripped}" - script_path_local = Path(script_dir) / script_filename + template_filename_stripped = template_filename + if prefix is not None: + script_filename = f"{prefix}{template_filename_stripped}" + else: + script_filename = template_filename_stripped + script_path_local = Path(script_dir) / script_filename # Read template customize_template( template_name=template_filename, replacements=replacements, script_path=script_path_local, ) - cmd = f"bash {script_path_local}" logger.debug(f"Now run '{cmd}' ") - stdout = execute_command_sync(command=cmd, logger_name=LOGGER_NAME) - logger.debug(f"Standard output of '{cmd}':\n{stdout}") logger.debug(f"_customize_and_run_template {template_filename} - END") - return stdout @@ -194,7 +196,7 @@ def collect_package_local( # Run script 1 stdout = _customize_and_run_template( - template_filename="_1_create_venv.sh", + template_filename="1_create_venv.sh", **common_args, ) activity.log = get_current_log(log_file_path) @@ -202,37 +204,29 @@ def collect_package_local( # Run script 2 stdout = _customize_and_run_template( - template_filename="_2_preliminary_pip_operations.sh", + template_filename="2_pip_install.sh", **common_args, ) activity.log = get_current_log(log_file_path) activity = add_commit_refresh(obj=activity, db=db) # Run script 3 - stdout = _customize_and_run_template( - template_filename="_3_pip_install.sh", - **common_args, - ) - activity.log = get_current_log(log_file_path) - activity = add_commit_refresh(obj=activity, db=db) - - # Run script 4 pip_freeze_stdout = _customize_and_run_template( - template_filename="_4_pip_freeze.sh", + template_filename="3_pip_freeze.sh", **common_args, ) activity.log = get_current_log(log_file_path) activity = add_commit_refresh(obj=activity, db=db) - # Run script 5 + # Run script 4 stdout = _customize_and_run_template( - template_filename="_5_pip_show.sh", + template_filename="4_pip_show.sh", **common_args, ) activity.log = get_current_log(log_file_path) activity = add_commit_refresh(obj=activity, db=db) - pkg_attrs = parse_script_5_stdout(stdout) + pkg_attrs = parse_script_pip_show_stdout(stdout) for key, value in pkg_attrs.items(): logger.debug(f"Parsed from pip-show: {key}={value}") # Check package_name match between pip show and task-group diff --git a/fractal_server/tasks/v2/ssh/collect.py b/fractal_server/tasks/v2/ssh/collect.py index d271b9cf6c..3315473b64 100644 --- a/fractal_server/tasks/v2/ssh/collect.py +++ b/fractal_server/tasks/v2/ssh/collect.py @@ -24,7 +24,9 @@ ) from fractal_server.tasks.v2.utils_templates import customize_template from fractal_server.tasks.v2.utils_templates import get_collection_replacements -from fractal_server.tasks.v2.utils_templates import parse_script_5_stdout +from fractal_server.tasks.v2.utils_templates import ( + parse_script_pip_show_stdout, +) from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER from fractal_server.utils import get_timestamp @@ -242,7 +244,7 @@ def collect_package_ssh( # Run script 1 stdout = _customize_and_run_template( - template_filename="_1_create_venv.sh", + template_filename="1_create_venv.sh", **common_args, ) activity.log = get_current_log(log_file_path) @@ -250,37 +252,30 @@ def collect_package_ssh( # Run script 2 stdout = _customize_and_run_template( - template_filename="_2_preliminary_pip_operations.sh", + template_filename="2_pip_install.sh", **common_args, ) activity.log = get_current_log(log_file_path) activity = add_commit_refresh(obj=activity, db=db) # Run script 3 - stdout = _customize_and_run_template( - template_filename="_3_pip_install.sh", - **common_args, - ) - activity.log = get_current_log(log_file_path) - activity = add_commit_refresh(obj=activity, db=db) - - # Run script 4 pip_freeze_stdout = _customize_and_run_template( - template_filename="_4_pip_freeze.sh", + template_filename="3_pip_freeze.sh", **common_args, ) activity.log = get_current_log(log_file_path) activity = add_commit_refresh(obj=activity, db=db) - # Run script 5 + # Run script 4 stdout = _customize_and_run_template( - template_filename="_5_pip_show.sh", + template_filename="4_pip_show.sh", **common_args, ) activity.log = get_current_log(log_file_path) activity = add_commit_refresh(obj=activity, db=db) - pkg_attrs = parse_script_5_stdout(stdout) + pkg_attrs = parse_script_pip_show_stdout(stdout) + for key, value in pkg_attrs.items(): logger.debug(f"parsed from pip-show: {key}={value}") # Check package_name match between pip show and task-group diff --git a/fractal_server/tasks/v2/templates/_1_create_venv.sh b/fractal_server/tasks/v2/templates/1_create_venv.sh similarity index 100% rename from fractal_server/tasks/v2/templates/_1_create_venv.sh rename to fractal_server/tasks/v2/templates/1_create_venv.sh diff --git a/fractal_server/tasks/v2/templates/_3_pip_install.sh b/fractal_server/tasks/v2/templates/2_pip_install.sh similarity index 79% rename from fractal_server/tasks/v2/templates/_3_pip_install.sh rename to fractal_server/tasks/v2/templates/2_pip_install.sh index 3a3cb98c3b..c25a201ed3 100644 --- a/fractal_server/tasks/v2/templates/_3_pip_install.sh +++ b/fractal_server/tasks/v2/templates/2_pip_install.sh @@ -5,16 +5,23 @@ write_log(){ echo "[collect-task, $TIMESTAMP] $1" } - # Variables to be filled within fractal-server PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__ INSTALL_STRING=__INSTALL_STRING__ PINNED_PACKAGE_LIST="__PINNED_PACKAGE_LIST__" +FRACTAL_MAX_PIP_VERSION="__FRACTAL_MAX_PIP_VERSION__" TIME_START=$(date +%s) VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python +# Upgrade `pip` and install `setuptools` +write_log "START upgrade pip and install setuptools" +"$VENVPYTHON" -m pip install --no-cache-dir "pip<=${FRACTAL_MAX_PIP_VERSION}" --upgrade +"$VENVPYTHON" -m pip install --no-cache-dir setuptools +write_log "END upgrade pip and install setuptools" +echo + # Install package write_log "START install ${INSTALL_STRING}" "$VENVPYTHON" -m pip install --no-cache-dir "$INSTALL_STRING" diff --git a/fractal_server/tasks/v2/templates/_4_pip_freeze.sh b/fractal_server/tasks/v2/templates/3_pip_freeze.sh similarity index 61% rename from fractal_server/tasks/v2/templates/_4_pip_freeze.sh rename to fractal_server/tasks/v2/templates/3_pip_freeze.sh index ef6e570a79..7a0e83f159 100644 --- a/fractal_server/tasks/v2/templates/_4_pip_freeze.sh +++ b/fractal_server/tasks/v2/templates/3_pip_freeze.sh @@ -1,12 +1,5 @@ set -e -write_log(){ - TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - echo "[collect-task, $TIMESTAMP] $1" -} - - - # Variables to be filled within fractal-server PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__ diff --git a/fractal_server/tasks/v2/templates/_5_pip_show.sh b/fractal_server/tasks/v2/templates/4_pip_show.sh similarity index 99% rename from fractal_server/tasks/v2/templates/_5_pip_show.sh rename to fractal_server/tasks/v2/templates/4_pip_show.sh index 52c8207d82..58c56307a0 100644 --- a/fractal_server/tasks/v2/templates/_5_pip_show.sh +++ b/fractal_server/tasks/v2/templates/4_pip_show.sh @@ -11,7 +11,6 @@ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__ PACKAGE_NAME=__PACKAGE_NAME__ - TIME_START=$(date +%s) VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python diff --git a/fractal_server/tasks/v2/templates/5_pip_install_from_freeze.sh b/fractal_server/tasks/v2/templates/5_pip_install_from_freeze.sh new file mode 100644 index 0000000000..18adb3b1bc --- /dev/null +++ b/fractal_server/tasks/v2/templates/5_pip_install_from_freeze.sh @@ -0,0 +1,35 @@ +set -e + +write_log(){ + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "[collect-task, $TIMESTAMP] $1" +} + +# Variables to be filled within fractal-server +PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__ +PIP_FREEZE_FILE=__PIP_FREEZE_FILE__ +FRACTAL_MAX_PIP_VERSION=__FRACTAL_MAX_PIP_VERSION__ + +TIME_START=$(date +%s) + +VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python + +# Upgrade `pip` and install `setuptools` +write_log "START upgrade pip and install setuptools" +"$VENVPYTHON" -m pip install --no-cache-dir "pip<=${FRACTAL_MAX_PIP_VERSION}" --upgrade +"$VENVPYTHON" -m pip install --no-cache-dir setuptools +write_log "END upgrade pip and install setuptools" +echo + +# Install from pip-freeze file +write_log "START installing requirements from ${PIP_FREEZE_FILE}" +"$VENVPYTHON" -m pip install -r "${PIP_FREEZE_FILE}" +write_log "END installing requirements from ${PIP_FREEZE_FILE}" +echo + +# End +TIME_END=$(date +%s) +write_log "All good up to here." +write_log "Elapsed: $((TIME_END - TIME_START)) seconds" +write_log "Exit." +echo diff --git a/fractal_server/tasks/v2/templates/_2_preliminary_pip_operations.sh b/fractal_server/tasks/v2/templates/_2_preliminary_pip_operations.sh deleted file mode 100644 index 8f3c1f0c18..0000000000 --- a/fractal_server/tasks/v2/templates/_2_preliminary_pip_operations.sh +++ /dev/null @@ -1,27 +0,0 @@ -set -e - -write_log(){ - TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - echo "[collect-task, $TIMESTAMP] $1" -} - -# Variables to be filled within fractal-server -PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__ - -TIME_START=$(date +%s) - -VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python - -# Upgrade pip -write_log "START upgrade pip" -"$VENVPYTHON" -m pip install --no-cache-dir "pip<=__FRACTAL_MAX_PIP_VERSION__" --upgrade -"$VENVPYTHON" -m pip install --no-cache-dir setuptools -write_log "END upgrade pip" -echo - -# End -TIME_END=$(date +%s) -write_log "All good up to here." -write_log "Elapsed: $((TIME_END - TIME_START)) seconds" -write_log "Exit." -echo diff --git a/fractal_server/tasks/v2/utils_templates.py b/fractal_server/tasks/v2/utils_templates.py index 3c6e973bcc..822baa7cb4 100644 --- a/fractal_server/tasks/v2/utils_templates.py +++ b/fractal_server/tasks/v2/utils_templates.py @@ -38,9 +38,9 @@ def customize_template( f.write(script_data) -def parse_script_5_stdout(stdout: str) -> dict[str, str]: +def parse_script_pip_show_stdout(stdout: str) -> dict[str, str]: """ - Parse standard output of template 5. + Parse standard output of 4_pip_show.sh """ searches = [ ("Python interpreter:", "python_bin"), diff --git a/tests/v2/06_tasks/test_unit_bash_scripts.py b/tests/v2/06_tasks/test_unit_bash_scripts.py index c421f05785..08bf225337 100644 --- a/tests/v2/06_tasks/test_unit_bash_scripts.py +++ b/tests/v2/06_tasks/test_unit_bash_scripts.py @@ -1,11 +1,14 @@ import pytest +from fractal_server.tasks.v2.local.collect import _customize_and_run_template from fractal_server.tasks.v2.utils_templates import customize_template -from fractal_server.tasks.v2.utils_templates import parse_script_5_stdout +from fractal_server.tasks.v2.utils_templates import ( + parse_script_pip_show_stdout, +) from fractal_server.utils import execute_command_sync -def test_parse_script_5_stdout(): +def test_parse_script_pip_show_stdout(): stdout = ( "Python interpreter: /some\n" "Package name: name\n" @@ -13,7 +16,7 @@ def test_parse_script_5_stdout(): "Package parent folder: /some\n" "Manifest absolute path: /some\n" ) - res = parse_script_5_stdout(stdout) + res = parse_script_pip_show_stdout(stdout) assert res == { "python_bin": "/some", "package_name": "name", @@ -31,10 +34,10 @@ def test_parse_script_5_stdout(): "Manifest absolute path: /some\n" ) with pytest.raises(ValueError, match="too many times"): - parse_script_5_stdout(stdout) + parse_script_pip_show_stdout(stdout) with pytest.raises(ValueError, match="not found"): - parse_script_5_stdout("invalid") + parse_script_pip_show_stdout("invalid") def test_template_1(tmp_path, current_py_version): @@ -46,7 +49,7 @@ def test_template_1(tmp_path, current_py_version): ] script_path = tmp_path / "1_good.sh" customize_template( - template_name="_1_create_venv.sh", + template_name="1_create_venv.sh", replacements=replacements, script_path=script_path.as_posix(), ) @@ -54,7 +57,7 @@ def test_template_1(tmp_path, current_py_version): assert venv_path.exists() -def test_template_3(tmp_path, testdata_path, current_py_version): +def test_template_2(tmp_path, testdata_path, current_py_version): path = tmp_path / "unit_templates" venv_path = path / "venv" install_string = testdata_path.parent / ( @@ -68,12 +71,12 @@ def test_template_3(tmp_path, testdata_path, current_py_version): replacements = [ ("__PACKAGE_ENV_DIR__", venv_path.as_posix()), ("__INSTALL_STRING__", install_string.as_posix()), - ("__PYTHON__", f"python{current_py_version}"), ("__PINNED_PACKAGE_LIST__", pinned_pkg_list), + ("__FRACTAL_MAX_PIP_VERSION__", "99"), ] - script_path = tmp_path / "3_good.sh" + script_path = tmp_path / "2_good.sh" customize_template( - template_name="_3_pip_install.sh", + template_name="2_pip_install.sh", replacements=replacements, script_path=script_path.as_posix(), ) @@ -90,12 +93,12 @@ def test_template_3(tmp_path, testdata_path, current_py_version): replacements = [ ("__PACKAGE_ENV_DIR__", venv_path_bad.as_posix()), ("__INSTALL_STRING__", install_string.as_posix()), - ("__PYTHON__", f"python{current_py_version}"), ("__PINNED_PACKAGE_LIST__", pinned_pkg_list), + ("__FRACTAL_MAX_PIP_VERSION__", "25"), ] - script_path = tmp_path / "3_bad_pkg.sh" + script_path = tmp_path / "2_bad_pkg.sh" customize_template( - template_name="_3_pip_install.sh", + template_name="2_pip_install.sh", replacements=replacements, script_path=script_path.as_posix(), ) @@ -104,7 +107,7 @@ def test_template_3(tmp_path, testdata_path, current_py_version): assert "Package(s) not found: pkgA" in str(expinfo.value) -def test_template_5(tmp_path, testdata_path, current_py_version): +def test_template_4(tmp_path, testdata_path, current_py_version): path = tmp_path / "unit_templates" venv_path = path / "venv" @@ -124,13 +127,11 @@ def test_template_5(tmp_path, testdata_path, current_py_version): ) replacements = [ ("__PACKAGE_ENV_DIR__", venv_path.as_posix()), - ("__INSTALL_STRING__", install_string.as_posix()), - ("__PYTHON__", f"python{current_py_version}"), ("__PACKAGE_NAME__", package_name), ] - script_path = tmp_path / "5_good.sh" + script_path = tmp_path / "4_good.sh" customize_template( - template_name="_5_pip_show.sh", + template_name="4_pip_show.sh", replacements=replacements, script_path=script_path.as_posix(), ) @@ -156,16 +157,92 @@ def test_template_5(tmp_path, testdata_path, current_py_version): ) replacements = [ ("__PACKAGE_ENV_DIR__", venv_path.as_posix()), - ("__INSTALL_STRING__", install_string_miss.as_posix()), - ("__PYTHON__", f"python{current_py_version}"), ("__PACKAGE_NAME__", package_name), ] - script_path = tmp_path / "5_good.sh" + script_path = tmp_path / "4_good.sh" customize_template( - template_name="_5_pip_show.sh", + template_name="4_pip_show.sh", replacements=replacements, script_path=script_path.as_posix(), ) with pytest.raises(RuntimeError) as expinfo: execute_command_sync(command=f"bash {script_path.as_posix()}") assert "ERROR: manifest path not found" in str(expinfo.value) + + +def _parse_pip_freeze_output(stdout: str) -> dict[str, str]: + splitted_output = stdout.split() + freeze_dict = dict([x.split("==") for x in splitted_output]) + return freeze_dict + + +def test_template_3_and_5(tmp_path, current_py_version): + + # Create two venvs + venv_path_1 = tmp_path / "venv1" + venv_path_2 = tmp_path / "venv2" + for venv_path in [venv_path_1, venv_path_2]: + _customize_and_run_template( + template_filename="1_create_venv.sh", + replacements=[ + ("__PACKAGE_ENV_DIR__", venv_path.as_posix()), + ("__PYTHON__", f"python{current_py_version}"), + ], + script_dir=tmp_path, + ) + _customize_and_run_template( + template_filename="2_pip_install.sh", + replacements=[ + ("__PACKAGE_ENV_DIR__", venv_path.as_posix()), + ("__INSTALL_STRING__", "pip"), + ("__FRACTAL_MAX_PIP_VERSION__", "99"), + ("__PINNED_PACKAGE_LIST__", ""), + ], + script_dir=tmp_path, + ) + + # Pip-install devtools on 'venv1' + _customize_and_run_template( + template_filename="2_pip_install.sh", + replacements=[ + ("__PACKAGE_ENV_DIR__", venv_path_1.as_posix()), + ("__INSTALL_STRING__", "devtools"), + ("__FRACTAL_MAX_PIP_VERSION__", "99"), + ("__PINNED_PACKAGE_LIST__", ""), + ], + script_dir=tmp_path, + ) + # Run script 3 (pip freeze) on 'venv1' + pip_freeze_venv_1 = _customize_and_run_template( + template_filename="3_pip_freeze.sh", + replacements=[("__PACKAGE_ENV_DIR__", venv_path_1.as_posix())], + script_dir=tmp_path, + ) + dependencies_1 = _parse_pip_freeze_output(pip_freeze_venv_1) + assert "pip" in dependencies_1 + + # Write requirements file + requirements_file = tmp_path / "requirements.txt" + with requirements_file.open("w") as f: + f.write(pip_freeze_venv_1) + + # Run script 5 (install from freeze) on 'venv2' + _customize_and_run_template( + template_filename="5_pip_install_from_freeze.sh", + replacements=[ + ("__PACKAGE_ENV_DIR__", venv_path_2.as_posix()), + ("__PIP_FREEZE_FILE__", requirements_file.as_posix()), + ("__FRACTAL_MAX_PIP_VERSION__", "99"), + ], + script_dir=tmp_path, + ) + + # Run script 3 (pip freeze) on 'venv2' + pip_freeze_venv_2 = _customize_and_run_template( + template_filename="3_pip_freeze.sh", + replacements=[("__PACKAGE_ENV_DIR__", venv_path_2.as_posix())], + script_dir=tmp_path, + ) + dependencies_2 = _parse_pip_freeze_output(pip_freeze_venv_2) + + assert dependencies_2 == dependencies_1