Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for logging into private docker registries #374

Merged
merged 5 commits into from
Jan 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions latch/types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from latch.types.file import LatchFile, LatchOutputFile
from latch.types.glob import file_glob
from latch.types.metadata import (
DockerMetadata,
Fork,
ForkBranch,
LatchAppearanceType,
Expand Down
7 changes: 7 additions & 0 deletions latch/types/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -521,10 +521,17 @@ def _parameter_str(t: Tuple[str, LatchParameter]):
).strip("\n ")


@dataclass
class DockerMetadata:
username: str
secret_name: str


@dataclass
class SnakemakeMetadata(LatchMetadata):
output_dir: Optional[LatchDir] = None
name: Optional[str] = None
docker_metadata: Optional[DockerMetadata] = None
parameters: Dict[str, SnakemakeParameter] = field(default_factory=dict)

def __post_init__(self):
Expand Down
32 changes: 19 additions & 13 deletions latch_cli/services/register/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -358,17 +358,21 @@ def register(
"N/A",
)
click.echo(
" ".join([
click.style("Target workspace:", fg="bright_blue"),
ws_name,
f"({current_workspace()})",
])
" ".join(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make sure your black version is up to date w/ what we have in pyproject.toml

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should not have any formatting diffs

[
click.style("Target workspace:", fg="bright_blue"),
ws_name,
f"({current_workspace()})",
]
)
)
click.echo(
" ".join([
click.style("Workflow root:", fg="bright_blue"),
str(ctx.default_container.pkg_dir),
])
" ".join(
[
click.style("Workflow root:", fg="bright_blue"),
str(ctx.default_container.pkg_dir),
]
)
)

if use_new_centromere:
Expand All @@ -386,10 +390,12 @@ def register(
scp = None

click.echo(
" ".join([
click.style("Docker Image:", fg="bright_blue"),
ctx.default_container.image_name,
])
" ".join(
[
click.style("Docker Image:", fg="bright_blue"),
ctx.default_container.image_name,
]
)
)
click.echo()

Expand Down
53 changes: 52 additions & 1 deletion latch_cli/snakemake/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,15 @@ def get_fn_code(
print(f"JIT Workflow Version: {{jit_wf_version}}")
print(f"JIT Execution Display Name: {{jit_exec_display_name}}")

wf = extract_snakemake_workflow(pkg_root, snakefile, jit_wf_version, jit_exec_display_name, local_to_remote_path_mapping, non_blob_parameters, {self.cache_tasks})
wf = extract_snakemake_workflow(
pkg_root,
snakefile,
jit_wf_version,
jit_exec_display_name,
local_to_remote_path_mapping,
non_blob_parameters,
{self.cache_tasks},
)
wf_name = wf.name
generate_snakemake_entrypoint(wf, pkg_root, snakefile, {repr(remote_output_url)}, non_blob_parameters)

Expand Down Expand Up @@ -659,6 +667,7 @@ def __init__(
self._input_parameters = None
self._dag = dag
self._cache_tasks = cache_tasks
self._docker_metadata = metadata._snakemake_metadata.docker_metadata
self.snakemake_tasks: List[SnakemakeJobTask] = []

workflow_metadata = WorkflowMetadata(
Expand Down Expand Up @@ -1385,6 +1394,48 @@ def get_fn_code(
except Exception:
traceback.print_exc()
lp.upload(compiled, "latch:///.snakemake_latch/workflows/{self.wf.name}/compiled_tasks/{self.name}.py")
""",
1,
)

if (
self.wf._docker_metadata is not None
and self.job.container_img_url is not None
):
code_block += reindent(
rf"""

print("\n\n\nLogging into Docker\n")
from latch.functions.secrets import get_secret
docker_usr = "{self.wf._docker_metadata.username}"
try:
docker_pwd = get_secret("{self.wf._docker_metadata.secret_name}")
except ValueError as e:
print("Failed to get Docker credentials:", e)
sys.exit(1)

try:
subprocess.run(
[
"docker",
"login",
"--username",
docker_usr,
"--password",
docker_pwd,
],
check=True,
)
except CalledProcessError as e:
print("Failed to login to Docker")
except Exception:
traceback.print_exc()
""",
1,
)

code_block += reindent(
rf"""

print("\n\n\nRunning snakemake task\n")
try:
Expand Down
Loading