Add the ability to use Github repos that contain symlinks.

src/zenml/integrations/github/code_repositories/github_code_repository.py

@@ -15,7 +15,7 @@
 
 import os
 import re
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 import requests
 from github import Github, GithubException
@@ -151,7 +151,7 @@ def download_files(
             raise RuntimeError("Invalid repository subdirectory.")
 
         os.makedirs(directory, exist_ok=True)
-
+        tmp_symlinks: List[Tuple[str, str]] = []
         for content in contents:
             local_path = os.path.join(directory, content.name)
             if content.type == "dir":
@@ -160,12 +160,25 @@ def download_files(
                     directory=local_path,
                     repo_sub_directory=content.path,
                 )
+            elif content.type == "symlink":
+                symlink_content = self.github_repo.get_contents(
+                    content.path, ref=commit
+                )
+                symlink_src = symlink_content.raw_data["target"]
+                tmp_symlinks.append((local_path, symlink_src))
+                # As it cannot be assumed at this point that the targets of the
+                # symlink already exist, the symlinks are first collected here and processed later.
             else:
                 try:
                     with open(local_path, "wb") as f:
                         f.write(content.decoded_content)
                 except (GithubException, IOError) as e:
                     logger.error("Error processing %s: %s", content.path, e)
+        for symlink in tmp_symlinks:
+            symlink_dst, symlink_src = symlink
+            create_symlink_in_local_repo_copy(
+                symlink_dst=symlink_dst, symlink_src=symlink_src
+            )
 
     def get_local_context(self, path: str) -> Optional[LocalRepositoryContext]:
         """Gets the local repository context.
@@ -202,3 +215,51 @@ def check_remote_url(self, url: str) -> bool:
             return True
 
         return False
+
+
+def create_symlink_in_local_repo_copy(
+    symlink_dst: str, symlink_src: str
+) -> None:
+    """This function attempts to create a symbolic link at `symlink_dst` that points to `symlink_src`.
+
+    If a file or directory already exists at `symlink_dst`, it will
+    be removed before the symbolic link is created.
+
+    Args:
+        symlink_dst: The path where the symbolic link should be created.
+        symlink_src: The path that the symbolic link should point to.
+
+    Raises:
+        FileNotFoundError: Informs that the target directory specified by
+            `symlink_dst` does not exist.
+        PermissionError: Informs that there are insufficient permissions to
+            create the symbolic link.
+        NotImplementedError: Informs that symbolic links are not supported on
+            the current operating system.
+        OSError: Any other OS-related errors that occur.
+    """
+    try:
+        os.symlink(src=symlink_src, dst=symlink_dst)
+
+    except FileExistsError as e:
+        logger.debug("The symbolic link already exists. %s",e)
+    except FileNotFoundError:
+        logger.warning(
+            "The target directory of the symbolic link '%s' does not exist. "
+            "The creation of the symbolic link is skipped.",
+            symlink_src,
+        )
+    except PermissionError:
+        logger.warning(
+            "You do not have the necessary permissions to create the symbolic link. "
+            "The creation of the symbolic link '%s' is skipped.",
+            symlink_dst,
+        )
+    except NotImplementedError:
+        logger.warning(
+            "Symbolic links are not supported on this operating system. "
+            "The creation of the symbolic link '%s' is skipped.",
+            symlink_dst,
+        )
+    except OSError as e:
+        logger.warning("An OS error occurred: %s", e)

tests/unit/integrations/github/test_github_code_repository.py

@@ -0,0 +1,47 @@
+import logging
+import os
+from tempfile import TemporaryDirectory
+
+import pytest
+
+from zenml.integrations.github.code_repositories.github_code_repository import (
+    create_symlink_in_local_repo_copy,
+)
+
+
+@pytest.fixture()
+def tmp_dir() -> str:
+    with TemporaryDirectory() as tmp:
+        yield tmp
+
+
+@pytest.mark.parametrize("initial_state", ["file", "directory"])
+def test_create_symlink_in_local_repo_copy(tmp_dir, initial_state: str):
+    symlink_dst = os.path.join(tmp_dir, "test_symlink")
+    symlink_src = os.path.join(tmp_dir, "target_folder")
+
+    if initial_state == "file":
+        with open(symlink_dst) as file:
+            file.write("content")
+    elif initial_state == "directory":
+        os.mkdir(symlink_dst)
+    os.mkdir(symlink_src)
+
+    create_symlink_in_local_repo_copy(
+        symlink_dst=symlink_dst, symlink_src=symlink_src
+    )
+    assert os.path.islink(symlink_dst)
+    assert os.readlink(symlink_dst) == str(symlink_src)
+
+
+def test_create_symlink_in_local_repo_copy_target_nonexistent(tmp_dir, caplog):
+    symlink_dst = os.path.join(tmp_dir, "test_symlink")
+    symlink_src = os.path.join(tmp_dir, "target_folder")
+
+    with caplog.at_level(logging.WARNING):
+        create_symlink_in_local_repo_copy(
+            symlink_dst=symlink_dst, symlink_src=symlink_src
+        )
+
+    assert not os.path.exists(symlink_dst)
+    assert "The target directory of the symbolic link" in caplog.text