From e151ccf9e456766779e3c3e06d7db1d482254323 Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 27 Sep 2024 18:02:47 +0800 Subject: [PATCH 1/3] feat: envd builder with copy commands Signed-off-by: mao3267 --- .../flytekitplugins/envd/image_builder.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index b11ab18bd1..b77796ec4f 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -176,6 +176,46 @@ def build(): else: envd_config += ' io.copy(source="./", target="/root")\n' + if image_spec.copy: + + def add_envd_copy_command(src_path: pathlib.Path, envd_config: str) -> str: + envd_version = metadata.version("envd") + if Version(envd_version) <= Version("0.3.37"): + if src_path.is_dir(): + envd_config += ( + f' io.copy(host_path="{src_path.as_posix()}", envd_path="/root/{src_path.as_posix()}/")\n' + ) + else: + envd_config += ( + f' io.copy(source="{src_path.as_posix()}", target="/root/{src_path.parent.as_posix()}/")\n' + ) + else: + if src_path.is_dir(): + envd_config += ( + f' io.copy(host_path="{src_path.as_posix()}", envd_path="/root/{src_path.as_posix()}/")\n' + ) + else: + envd_config += ( + f' io.copy(source="{src_path.as_posix()}", target="/root/{src_path.parent.as_posix()}/")\n' + ) + return envd_config + + for src in image_spec.copy: + src_path = pathlib.Path(src) + + if src_path.is_absolute() or ".." in src_path.parts: + raise ValueError("Absolute paths or paths with '..' are not allowed in COPY command.") + + dst_path = pathlib.Path(cfg_path).parent / src_path + dst_path.parent.mkdir(parents=True, exist_ok=True) + + if src_path.is_dir(): + shutil.copytree(src_path, dst_path, dirs_exist_ok=True) + else: + shutil.copy(src_path, dst_path) + + envd_config = add_envd_copy_command(src_path, envd_config) + with open(cfg_path, "w+") as f: f.write(envd_config) From 8392a514a8fe1ff9df0ad22537cc73712839330d Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 27 Sep 2024 18:04:07 +0800 Subject: [PATCH 2/3] test: envd config and build test Signed-off-by: mao3267 --- .../flytekit-envd/tests/test_image_spec.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/plugins/flytekit-envd/tests/test_image_spec.py b/plugins/flytekit-envd/tests/test_image_spec.py index c7db2f3cb9..702da6baf6 100644 --- a/plugins/flytekit-envd/tests/test_image_spec.py +++ b/plugins/flytekit-envd/tests/test_image_spec.py @@ -6,7 +6,7 @@ from flytekitplugins.envd.image_builder import EnvdImageSpecBuilder, create_envd_config from flytekit.image_spec.image_spec import ImageBuildEngine, ImageSpec - +import tempfile @pytest.fixture(scope="module", autouse=True) def register_envd_higher_priority(): @@ -32,24 +32,29 @@ def test_image_spec(): # so Envd can find the base image when building imageSpec below ImageBuildEngine._IMAGE_NAME_TO_REAL_NAME[base_image.image_name()] = "cr.flyte.org/flyteorg/flytekit:py3.8-latest" - image_spec = ImageSpec( - packages=["pandas"], - apt_packages=["git"], - python_version="3.8", - base_image=base_image, - pip_index="https://pypi.python.org/simple", - source_root=os.path.dirname(os.path.realpath(__file__)), - ) - - image_spec = image_spec.with_commands("echo hello") - ImageBuildEngine.build(image_spec) - image_spec.base_image = base_image.image_name() - config_path = create_envd_config(image_spec) - assert image_spec.platform == "linux/amd64" - contents = Path(config_path).read_text() - assert ( - contents - == f"""# syntax=v1 + with tempfile.TemporaryDirectory(dir=Path.cwd().as_posix()) as temp_dir: + copy_file = Path(temp_dir) / "copy_file.txt" + copy_file.write_text("copy_file_content") + + image_spec = ImageSpec( + packages=["pandas"], + apt_packages=["git"], + python_version="3.8", + base_image=base_image, + pip_index="https://pypi.python.org/simple", + source_root=os.path.dirname(os.path.realpath(__file__)), + copy=[f"{copy_file.relative_to(Path.cwd()).as_posix()}"], + ) + + image_spec = image_spec.with_commands("echo hello") + ImageBuildEngine.build(image_spec) + image_spec.base_image = base_image.image_name() + config_path = create_envd_config(image_spec) + assert image_spec.platform == "linux/amd64" + contents = Path(config_path).read_text() + assert ( + contents + == f"""# syntax=v1 def build(): base(image="cr.flyte.org/flyteorg/flytekit:py3.8-latest", dev=False) @@ -60,6 +65,7 @@ def build(): config.pip_index(url="https://pypi.python.org/simple") install.python(version="3.8") io.copy(source="./", target="/root") + io.copy(source="{copy_file.relative_to(Path.cwd()).as_posix()}", target="/root/{copy_file.parent.relative_to(Path.cwd()).as_posix()}/") """ ) From aca164ad7b14e85e93d5d80ca3f6716f26c4f6cf Mon Sep 17 00:00:00 2001 From: mao3267 Date: Tue, 1 Oct 2024 21:15:03 +0800 Subject: [PATCH 3/3] fix: copy command for different envd versions Signed-off-by: mao3267 --- plugins/flytekit-envd/flytekitplugins/envd/image_builder.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index b77796ec4f..bc8ef7de68 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -186,13 +186,11 @@ def add_envd_copy_command(src_path: pathlib.Path, envd_config: str) -> str: f' io.copy(host_path="{src_path.as_posix()}", envd_path="/root/{src_path.as_posix()}/")\n' ) else: - envd_config += ( - f' io.copy(source="{src_path.as_posix()}", target="/root/{src_path.parent.as_posix()}/")\n' - ) + envd_config += f' io.copy(host_path="{src_path.as_posix()}", envd_path="/root/{src_path.parent.as_posix()}/")\n' else: if src_path.is_dir(): envd_config += ( - f' io.copy(host_path="{src_path.as_posix()}", envd_path="/root/{src_path.as_posix()}/")\n' + f' io.copy(source="{src_path.as_posix()}", target="/root/{src_path.as_posix()}/")\n' ) else: envd_config += (