Run metrics workflows as part of main module WDLs (#211)

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module00a.json.tmpl

@@ -26,6 +26,9 @@
   "Module00a.gatk_docker_pesr_override": "${workspace.gatk_docker_pesr_override}",
   "Module00a.cloud_sdk_docker": "${workspace.cloud_sdk_docker}",
 
+  "Module00a.primary_contigs_fai": "${workspace.primary_contigs_fai}",
+  "Module00a.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+
   "Module00a.bam_or_cram_file": "${this.bam_or_cram_file}",
   "Module00a.sample_id": "${this.sample_id}",
 

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module01.json.tmpl

@@ -13,6 +13,10 @@
   "Module01.depth_exclude_list": "${workspace.depth_exclude_list}",
   "Module01.depth_exclude_list_frac_max": "0.5",
 
+  "Module01.primary_contigs_list": "${workspace.primary_contigs_list}",
+  "Module01.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+  "Module01.linux_docker": "${workspace.linux_docker}",
+
   "Module01.batch": "${this.sample_set_id}",
   "Module01.del_bed": "${this.merged_dels}",
   "Module01.dup_bed": "${this.merged_dups}",

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module02.json.tmpl

@@ -17,6 +17,8 @@
   "Module02.segdups": "${workspace.segdups}",
   "Module02.ref_dict": "${workspace.reference_dict}",
 
+  "Module02.primary_contigs_list": "${workspace.primary_contigs_list}",
+
   "Module02.batch": "${this.sample_set_id}",
   "Module02.ped_file": "${workspace.cohort_ped_file}",
 

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module03.json.tmpl

@@ -5,6 +5,10 @@
 
   "Module03.outlier_cutoff_nIQR": "10000",
 
+  "Module03.primary_contigs_list": "${workspace.primary_contigs_list}",
+  "Module03.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+  "Module03.ped_file": "${workspace.cohort_ped_file}",
+
   "Module03.batch": "${this.sample_set_id}",
   "Module03.depth_vcf" : "${this.clustered_depth_vcf}",
   "Module03.manta_vcf" : "${this.clustered_manta_vcf}",

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module04.SingleBatch.json.tmpl

@@ -11,6 +11,9 @@
   "Module04.reference_build": "${workspace.reference_build}",
   "Module04.ref_dict": "${workspace.reference_dict}",
 
+  "Module04.primary_contigs_list": "${workspace.primary_contigs_list}",
+  "Module04.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+
   "Module04.batch": "${this.sample_set_id}",
   "Module04.rf_cutoffs": "${this.cutoffs}",
   "Module04.batch_depth_vcf": "${this.filtered_depth_vcf}",

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module04.json.tmpl

@@ -11,6 +11,9 @@
   "Module04.reference_build": "${workspace.reference_build}",
   "Module04.ref_dict": "${workspace.reference_dict}",
 
+  "Module04.primary_contigs_list": "${workspace.primary_contigs_list}",
+  "Module04.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+
   "Module04.batch": "${this.sample_set_id}",
   "Module04.rf_cutoffs": "${this.cutoffs}",
   "Module04.batch_depth_vcf": "${this.filtered_depth_vcf}",

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module0506.SingleBatch.json.tmpl

@@ -25,6 +25,9 @@
   "Module0506.sv_pipeline_rdtest_docker": "${workspace.sv_pipeline_rdtest_docker}",
   "Module0506.sv_pipeline_qc_docker": "${workspace.sv_pipeline_qc_docker}",
 
+  "Module0506.primary_contigs_list": "${workspace.primary_contigs_list}",
+  "Module0506.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+
   "Module0506.cohort_name": "${this.sample_set_id}",
   "Module0506.batches": "${this.sample_set_id}",
   "Module0506.ped_file": "${workspace.cohort_ped_file}",

input_templates/terra_workspaces/cohort_mode/workflow_configurations/Module0506.json.tmpl

@@ -25,6 +25,9 @@
   "Module0506.sv_pipeline_rdtest_docker": "${workspace.sv_pipeline_rdtest_docker}",
   "Module0506.sv_pipeline_qc_docker": "${workspace.sv_pipeline_qc_docker}",
 
+  "Module0506.primary_contigs_list": "${workspace.primary_contigs_list}",
+  "Module0506.sv_pipeline_base_docker": "${workspace.sv_pipeline_base_docker}",
+
   "Module0506.cohort_name": "${this.sample_set_set_id}",
   "Module0506.batches": "${this.sample_sets.sample_set_id}",
   "Module0506.ped_file": "${workspace.cohort_ped_file}",

scripts/test/terra_validation.py

@@ -8,8 +8,8 @@
 import os.path
 
 """
-Summary: Cursory validation of Terra input JSONs for pipeline workflows and
-    metrics workflows. Checks if JSONs contain all required inputs and that
+Summary: Cursory validation of Terra input JSONs for pipeline workflows.
+    Checks if JSONs contain all required inputs and that
     they do not contain extraneous inputs. Does not perform type-checking.
     Also checks that all expected JSONs have been generated.
 
@@ -21,9 +21,7 @@
     /path/to/womtool/jar: path to user's womtool jar file
     Optional inputs:
     -n,--num-input-jsons INT: override default expected number of Terra input
-        JSONs (excluding metrics workflows)
-    -m,--num-metrics-jsons INT: override default expected number of Terra
-        metrics input JSONs
+        JSONs
     --log-level LEVEL: specify level of logging information to print,
         ie. INFO, WARNING, ERROR - not case-sensitive)
 
@@ -34,10 +32,9 @@
 
 WDLS_PATH = "wdl/"
 TERRA_INPUTS_PATH = "inputs/terra_workspaces/cohort_mode/workflow_configurations/"
-METRICS_INPUTS_PATH = "metrics_workflows/"
 
 
-def list_some_jsons(inputs_path, expected_num_jsons, subdir="", description=""):
+def list_jsons(inputs_path, expected_num_jsons, subdir="", description=""):
     jsons = [os.path.join(subdir, x) for x in listdir(os.path.join(inputs_path, subdir)) if x.endswith(".json")]
     num_input_jsons = len(jsons)
     if num_input_jsons < expected_num_jsons:
@@ -46,16 +43,8 @@ def list_some_jsons(inputs_path, expected_num_jsons, subdir="", description=""):
     return jsons
 
 
-def list_all_jsons(inputs_path, expected_num_inputs, expected_num_metrics):
-    jsons = list_some_jsons(inputs_path, expected_num_inputs)
-    metrics_jsons = list_some_jsons(inputs_path, expected_num_metrics, METRICS_INPUTS_PATH, "metrics ")
-
-    jsons.extend(metrics_jsons)
-    return jsons
-
-
-def get_wdl_json_pairs(wdl_path, terra_inputs_path, expected_num_inputs, expected_num_metrics):
-    jsons = list_all_jsons(terra_inputs_path, expected_num_inputs, expected_num_metrics)
+def get_wdl_json_pairs(wdl_path, terra_inputs_path, expected_num_inputs):
+    jsons = list_jsons(terra_inputs_path, expected_num_inputs)
 
     for json_file in jsons:
         path_to_wdl = os.path.join(wdl_path, os.path.basename(json_file).split(".")[0] + ".wdl")
@@ -93,20 +82,18 @@ def validate_terra_json(wdl, terra_json, womtool_jar):
     return int(valid)  # return 1 if valid, 0 if not valid
 
 
-def validate_all_terra_jsons(base_dir, womtool_jar, expected_num_inputs, expected_num_metrics):
+def validate_all_terra_jsons(base_dir, womtool_jar, expected_num_inputs):
     successes = 0
     for wdl, json_file in get_wdl_json_pairs(os.path.join(base_dir, WDLS_PATH),
                                              os.path.join(base_dir, TERRA_INPUTS_PATH),
-                                             expected_num_inputs,
-                                             expected_num_metrics):
+                                             expected_num_inputs):
         successes += validate_terra_json(wdl, json_file, womtool_jar)
 
     print("\n")
-    total_jsons = expected_num_inputs + expected_num_metrics
-    if successes != total_jsons:
-        raise RuntimeError(f"{successes} of {total_jsons} Terra input JSONs passed validation.")
+    if successes != expected_num_inputs:
+        raise RuntimeError(f"{successes} of {expected_num_inputs} Terra input JSONs passed validation.")
     else:
-        print(f"{successes} of {total_jsons} Terra input JSONs passed validation.")
+        print(f"{successes} of {expected_num_inputs} Terra input JSONs passed validation.")
 
 
 # Main function
@@ -115,25 +102,23 @@ def main():
     parser.add_argument("-d", "--base-dir", help="Relative path to base of gatk-sv repo", required=True)
     parser.add_argument("-j", "--womtool-jar", help="Path to womtool jar", required=True)
     parser.add_argument("-n", "--num-input-jsons",
-                        help="Number of Terra input JSONs expected (excluding metrics workflows)",
+                        help="Number of Terra input JSONs expected",
                         required=False, default=16, type=int)
-    parser.add_argument("-m", "--num-metrics-jsons", help="Number of Terra metrics workflow input JSONs expected",
-                        required=False, default=7, type=int)
     parser.add_argument("--log-level",
                         help="Specify level of logging information, ie. info, warning, error (not case-sensitive)",
                         required=False, default="INFO")
     args = parser.parse_args()
 
     # get args as variables
     base_dir, womtool_jar, log_level = args.base_dir, args.womtool_jar, args.log_level
-    expected_num_inputs, expected_num_metrics = args.num_input_jsons, args.num_metrics_jsons
+    expected_num_inputs = args.num_input_jsons
 
     numeric_level = getattr(logging, log_level.upper(), None)
     if not isinstance(numeric_level, int):
         raise ValueError('Invalid log level: %s' % log_level)
     logging.basicConfig(level=numeric_level, format='%(levelname)s: %(message)s')
 
-    validate_all_terra_jsons(base_dir, womtool_jar, expected_num_inputs, expected_num_metrics)
+    validate_all_terra_jsons(base_dir, womtool_jar, expected_num_inputs)
 
 
 if __name__ == "__main__":

test_input_templates/module00a/Module00aBatch.json.tmpl

@@ -28,6 +28,11 @@
   "Module00aBatch.gatk_docker_pesr_override" : {{ dockers.gatk_docker | tojson }},
   "Module00aBatch.cloud_sdk_docker": {{ dockers.cloud_sdk_docker | tojson }},
 
+  "Module00aBatch.batch": {{ test_batch.batch_name | tojson }},
+  "Module00aBatch.primary_contigs_fai": {{ reference_resources.primary_contigs_fai | tojson }},
+  "Module00aBatch.sv_pipeline_base_docker": {{ dockers.sv_pipeline_base_docker | tojson }},
+  "Module00aBatch.linux_docker": {{ dockers.linux_docker | tojson }},
+
   "Module00aBatch.bam_or_cram_files": {{ test_batch.bam_or_cram_files | tojson }},
   "Module00aBatch.sample_ids": {{ test_batch.samples | tojson }}
 }