Skip to content

Commit

Permalink
conflcit
Browse files Browse the repository at this point in the history
  • Loading branch information
kennyworkman committed Dec 1, 2023
1 parent 965a74f commit c467709
Show file tree
Hide file tree
Showing 10 changed files with 1,682 additions and 19 deletions.
14 changes: 14 additions & 0 deletions latch/types/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,4 +537,18 @@ def __post_init__(self):
_snakemake_metadata = self


@dataclass
class NextflowMetadata(LatchMetadata):
name: Optional[str] = None
parameters: Dict[str, SnakemakeParameter] = field(default_factory=dict)

def __post_init__(self):
if self.name is None:
self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"

global _nextflow_metadata
_nextflow_metadata = self


_snakemake_metadata: Optional[SnakemakeMetadata] = None
_nextflow_metadata: Optional[NextflowMetadata] = None
62 changes: 50 additions & 12 deletions latch_cli/centromere/ctx.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ class _CentromereCtx:
default_container: _Container
workflow_type: WorkflowType
snakefile: Optional[Path]
nf_script: Optional[Path]

latch_register_api_url = config.api.workflow.register
latch_image_api_url = config.api.workflow.upload_image
Expand All @@ -82,6 +83,7 @@ def __init__(
disable_auto_version: bool = False,
remote: bool = False,
snakefile: Optional[Path] = None,
nf_script: Optional[Path] = None,
use_new_centromere: bool = False,
):
self.use_new_centromere = use_new_centromere
Expand All @@ -95,11 +97,20 @@ def __init__(
self.dkr_repo = config.dkr_repo
self.pkg_root = pkg_root.resolve()

if snakefile is None:
self.workflow_type = WorkflowType.latchbiosdk
else:
if snakefile and nf_script:
raise ValueError(
"Cannot provide both a snakefile and nextflow script to the"
" register command."
)

if snakefile is not None:
self.workflow_type = WorkflowType.snakemake
self.snakefile = snakefile
elif nf_script is not None:
self.workflow_type = WorkflowType.nextflow
self.nf_script = nf_script
else:
self.workflow_type = WorkflowType.latchbiosdk

self.container_map: Dict[str, _Container] = {}
if self.workflow_type == WorkflowType.latchbiosdk:
Expand Down Expand Up @@ -128,7 +139,7 @@ def __init__(
fg="red",
)
raise click.exceptions.Exit(1)
else:
elif self.workflow_type == WorkflowType.snakemake:
assert snakefile is not None

import latch.types.metadata as metadata
Expand Down Expand Up @@ -166,11 +177,13 @@ def __init__(
fg="red",
)
click.secho(
"\nIt is possible to avoid including the Snakefile"
" prior to registration by providing a"
" `latch_metadata.py` file in the workflow root.\nThis"
" way it is not necessary to install dependencies or"
" ensure that Snakemake inputs locally.",
(
"\nIt is possible to avoid including the Snakefile"
" prior to registration by providing a"
" `latch_metadata.py` file in the workflow root.\nThis"
" way it is not necessary to install dependencies or"
" ensure that Snakemake inputs locally."
),
fg="red",
)
click.secho("\nExample ", fg="red", nl=False)
Expand Down Expand Up @@ -240,6 +253,28 @@ def __init__(
# name for snakemake
self.workflow_name = metadata._snakemake_metadata.name

else:
assert nf_script is not None

import latch.types.metadata as metadata

meta = pkg_root / "latch_metadata" / "__init__.py"
if meta.exists():
click.echo(f"Using metadata file {click.style(meta, italic=True)}")
import_module_by_path(meta)

if metadata._nextflow_metadata is None:
click.secho(
dedent(
"""Make sure a `latch_metadata.py` file exists in the
nextflow project root.""",
),
fg="red",
)
raise click.exceptions.Exit(1)

self.workflow_name = metadata._nextflow_metadata.name

version_file = self.pkg_root / "version"
try:
self.version = version_file.read_text()
Expand All @@ -258,8 +293,10 @@ def __init__(

if self.nucleus_check_version(self.version, self.workflow_name):
click.secho(
f"\nVersion ({self.version}) already exists."
" Make sure that you've saved any changes you made.",
(
f"\nVersion ({self.version}) already exists."
" Make sure that you've saved any changes you made."
),
fg="red",
bold=True,
)
Expand Down Expand Up @@ -298,7 +335,8 @@ def __init__(
)
self.ssh_client = ssh_client

def _patched_connect(self): ...
def _patched_connect(self):
...

def _patched_create_paramiko_client(self, base_url):
self.ssh_client = ssh_client
Expand Down
13 changes: 12 additions & 1 deletion latch_cli/docker_utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ def get_prologue(
) -> List[str]:
if wf_type == WorkflowType.snakemake:
library_name = '"latch[snakemake]"'
elif wf_type == WorkflowType.nextflow:
library_name = '"latch[nextflow]"'
else:
library_name = "latch"
return [
Expand Down Expand Up @@ -80,6 +82,14 @@ def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
"# DO NOT CHANGE",
"copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
]
elif wf_type == WorkflowType.nextflow:
cmds += [
"",
"# Latch nextflow workflow entrypoint",
"# DO NOT CHANGE",
"copy .latch/latch-nextflow /root/latch-nextflow",
"copy .latch/nextflow_jit_entrypoint.py /root/nextflow_jit_entrypoint.py",
]

cmds += [
"",
Expand Down Expand Up @@ -248,7 +258,8 @@ def infer_commands(pkg_root: Path) -> List[DockerCmdBlock]:

has_buildable_pyproject = True
break
except FileNotFoundError: ...
except FileNotFoundError:
...

# from https://peps.python.org/pep-0518/ and https://peps.python.org/pep-0621/
if has_setup_py or has_buildable_pyproject:
Expand Down
12 changes: 10 additions & 2 deletions latch_cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,10 @@ def dockerfile(pkg_root: str, snakemake: bool = False):
"Whether or not to cache snakemake tasks. Ignored if --snakefile is not"
" provided."
),
"--nf-script",
type=click.Path(exists=True, dir_okay=False, path_type=Path),
default=None,
help="Path to a nextflow script to register.",
)
@requires_login
def register(
Expand All @@ -186,6 +190,7 @@ def register(
yes: bool,
snakefile: Optional[Path],
cache_tasks: bool,
nf_script: Optional[Path],
):
"""Register local workflow code to Latch.
Expand All @@ -205,6 +210,7 @@ def register(
remote=remote,
skip_confirmation=yes,
snakefile=snakefile,
nf_script=nf_script,
progress_plain=(docker_progress == "auto" and not sys.stdout.isatty())
or docker_progress == "plain",
use_new_centromere=use_new_centromere,
Expand Down Expand Up @@ -523,8 +529,10 @@ def get_params(wf_name: Union[str, None], version: Union[str, None] = None):
if version is None:
version = "latest"
click.secho(
f"Successfully generated python param map named {wf_name}.params.py with"
f" version {version}\n Run `latch launch {wf_name}.params.py` to launch it.",
(
f"Successfully generated python param map named {wf_name}.params.py with"
f" version {version}\n Run `latch launch {wf_name}.params.py` to launch it."
),
fg="green",
)

Expand Down
Empty file added latch_cli/nextflow/__init__.py
Empty file.
Loading

0 comments on commit c467709

Please sign in to comment.