Eliminate Array[File?] workflow outputs not supported by Terra (#222)

input_templates/terra_workspaces/cohort_mode/workflow_configurations/FilterBatchSamples.json.tmpl

@@ -6,6 +6,9 @@
   "FilterBatchSamples.N_IQR_cutoff": "10000",
 
   "FilterBatchSamples.batch": "${this.sample_set_id}",
-  "FilterBatchSamples.vcfs" : "${this.sites_filtered_vcfs}",
-  "FilterBatchSamples.sv_counts": "${this.sv_counts}"
+  "FilterBatchSamples.manta_vcf" : "${this.sites_filtered_manta_vcf}",
+  "FilterBatchSamples.delly_vcf" : "${this.sites_filtered_delly_vcf}",
+  "FilterBatchSamples.wham_vcf" : "${this.sites_filtered_wham_vcf}",
+  "FilterBatchSamples.melt_vcf" : "${this.sites_filtered_melt_vcf}",
+  "FilterBatchSamples.depth_vcf" : "${this.sites_filtered_depth_vcf}"
 }

input_templates/terra_workspaces/cohort_mode/workflow_configurations/PlotSVCountsPerSample.json.tmpl

@@ -4,6 +4,6 @@
   "PlotSVCountsPerSample.N_IQR_cutoff": "6",
 
   "PlotSVCountsPerSample.prefix": "${this.sample_set_id}",
-  "PlotSVCountsPerSample.vcfs" : "${this.sites_filtered_vcfs}",
+  "PlotSVCountsPerSample.vcfs" : "${[this.sites_filtered_manta_vcf, this.sites_filtered_delly_vcf, this.sites_filtered_wham_vcf, this.sites_filtered_melt_vcf, this.sites_filtered_depth_vcf]}",
   "PlotSVCountsPerSample.vcf_identifiers" : "${this.algorithms_filtersites}"
 }

test_input_templates/FilterBatch/FilterBatchSamples.json.tmpl

@@ -7,18 +7,12 @@
   "FilterBatchSamples.outlier_cutoff_table" : {{ test_batch.outlier_cutoff_table | tojson }},
 
   "FilterBatchSamples.batch": {{ test_batch.batch_name | tojson }},
-  "FilterBatchSamples.vcfs" : [ 
-    {{ test_batch.sites_filtered_manta_vcf | tojson }}, 
-    null,
-    {{ test_batch.sites_filtered_wham_vcf | tojson }}, 
-    {{ test_batch.sites_filtered_melt_vcf | tojson }}, 
-    {{ test_batch.sites_filtered_depth_vcf | tojson }}
-  ],
-  "FilterBatchSamples.sv_counts": [
-    {{ test_batch.sites_filtered_svcounts_manta | tojson }},
-    null,
-    {{ test_batch.sites_filtered_svcounts_wham | tojson }},
-    {{ test_batch.sites_filtered_svcounts_melt | tojson }},
-    {{ test_batch.sites_filtered_svcounts_depth | tojson }}
-  ]
+  "FilterBatchSamples.manta_vcf": {{ test_batch.sites_filtered_manta_vcf | tojson }},
+  "FilterBatchSamples.wham_vcf": {{ test_batch.sites_filtered_wham_vcf | tojson }},
+  "FilterBatchSamples.melt_vcf": {{ test_batch.sites_filtered_melt_vcf | tojson }},
+  "FilterBatchSamples.depth_vcf": {{ test_batch.sites_filtered_depth_vcf | tojson }},
+  "FilterBatchSamples.manta_counts": {{ test_batch.sites_filtered_svcounts_manta | tojson }},
+  "FilterBatchSamples.wham_counts": {{ test_batch.sites_filtered_svcounts_wham | tojson }},
+  "FilterBatchSamples.melt_counts": {{ test_batch.sites_filtered_svcounts_melt | tojson }},
+  "FilterBatchSamples.depth_counts": {{ test_batch.sites_filtered_svcounts_depth | tojson }}
 }

wdl/FilterBatch.wdl

@@ -68,7 +68,7 @@ workflow FilterBatch {
   call sv_counts.PlotSVCountsPerSample {
     input:
       prefix = batch,
-      vcfs = FilterBatchSites.sites_filtered_vcfs,
+      vcfs = [FilterBatchSites.sites_filtered_manta_vcf, FilterBatchSites.sites_filtered_delly_vcf, FilterBatchSites.sites_filtered_wham_vcf, FilterBatchSites.sites_filtered_melt_vcf, FilterBatchSites.sites_filtered_depth_vcf],
       vcf_identifiers = FilterBatchSites.algorithms_filtersites,
       N_IQR_cutoff = outlier_cutoff_nIQR,
       sv_pipeline_docker = sv_pipeline_docker,
@@ -82,8 +82,11 @@ workflow FilterBatch {
       batch = batch,
       outlier_cutoff_table = outlier_cutoff_table,
       N_IQR_cutoff = outlier_cutoff_nIQR,
-      vcfs = FilterBatchSites.sites_filtered_vcfs,
-      sv_counts = PlotSVCountsPerSample.sv_counts,
+      manta_vcf = FilterBatchSites.sites_filtered_manta_vcf,
+      delly_vcf = FilterBatchSites.sites_filtered_delly_vcf,
+      wham_vcf = FilterBatchSites.sites_filtered_wham_vcf,
+      melt_vcf = FilterBatchSites.sites_filtered_melt_vcf,
+      depth_vcf = FilterBatchSites.sites_filtered_depth_vcf,
       linux_docker = linux_docker,
       sv_pipeline_docker = sv_pipeline_docker,
       sv_base_mini_docker = sv_base_mini_docker,
@@ -133,8 +136,8 @@ workflow FilterBatch {
     File cutoffs = FilterBatchSites.cutoffs
     File scores = FilterBatchSites.scores
     File RF_intermediate_files = FilterBatchSites.RF_intermediate_files
-    Array[File?] sv_counts = PlotSVCountsPerSample.sv_counts
-    Array[File?] sv_count_plots = PlotSVCountsPerSample.sv_count_plots
+    Array[File] sv_counts = PlotSVCountsPerSample.sv_counts
+    Array[File] sv_count_plots = PlotSVCountsPerSample.sv_count_plots
     Array[String] outlier_samples_excluded = FilterBatchSamples.outlier_samples_excluded
     Array[String] batch_samples_postOutlierExclusion = FilterBatchSamples.filtered_batch_samples_list
     File outlier_samples_excluded_file = FilterBatchSamples.outlier_samples_excluded_file

wdl/FilterBatchSamples.wdl

@@ -9,8 +9,16 @@ import "IdentifyOutlierSamples.wdl" as identify_outliers
 workflow FilterBatchSamples {
   input {
     String batch
-    Array[File?] vcfs  # in order of algorithms array: ["manta", "delly", "wham", "melt", "depth"]. To skip one, use null keyword - ex. ["manta.vcf.gz", null, "wham.vcf.gz", null, "depth.vcf.gz"]
-    Array[File?] sv_counts  # one SV counts file from PlotSVCountsPerSample per VCF in the same order
+    File? manta_vcf
+    File? delly_vcf
+    File? wham_vcf
+    File? melt_vcf
+    File? depth_vcf
+    File? manta_counts  # SV counts files from PlotSVCountsPerSample. If not provided, SV counts will be calculated as part of this workflow
+    File? delly_counts
+    File? wham_counts
+    File? melt_counts
+    File? depth_counts
     Int N_IQR_cutoff
     File? outlier_cutoff_table
     String sv_pipeline_docker
@@ -25,16 +33,18 @@ workflow FilterBatchSamples {
     RuntimeAttr? runtime_attr_merge_pesr_vcfs
   }
 
+  Array[File?] vcfs = [manta_vcf, delly_vcf, wham_vcf, melt_vcf, depth_vcf]
   Array[String] algorithms = ["manta", "delly", "wham", "melt", "depth"]  # fixed algorithms to enable File outputs to be determined
   Int num_algorithms = length(algorithms)
+  Array[File?] sv_counts_ = [manta_counts, delly_counts, wham_counts, melt_counts, depth_counts]
 
   scatter (i in range(num_algorithms)) {
     if (defined(vcfs[i])) {
       call identify_outliers.IdentifyOutlierSamples {
         input:
           vcf = select_first([vcfs[i]]),
           name = batch,
-          sv_counts = sv_counts[i], 
+          sv_counts = sv_counts_[i],
           N_IQR_cutoff = N_IQR_cutoff,
           outlier_cutoff_table = outlier_cutoff_table,
           vcf_identifier = algorithms[i],

wdl/FilterBatchSites.wdl

@@ -59,7 +59,11 @@ workflow FilterBatchSites {
   }
 
   output {
-    Array[File?] sites_filtered_vcfs = FilterAnnotateVcf.annotated_vcf
+    File? sites_filtered_manta_vcf = FilterAnnotateVcf.annotated_vcf[0]
+    File? sites_filtered_delly_vcf = FilterAnnotateVcf.annotated_vcf[1]
+    File? sites_filtered_wham_vcf = FilterAnnotateVcf.annotated_vcf[2]
+    File? sites_filtered_melt_vcf = FilterAnnotateVcf.annotated_vcf[3]
+    File? sites_filtered_depth_vcf = FilterAnnotateVcf.annotated_vcf[4]
     File cutoffs = AdjudicateSV.cutoffs
     File scores = RewriteScores.updated_scores
     File RF_intermediate_files = AdjudicateSV.RF_intermediate_files

wdl/PlotSVCountsPerSample.wdl

@@ -49,8 +49,8 @@ workflow PlotSVCountsPerSample {
 
 
   output {
-    Array[File?] sv_counts = CountSVsPerSamplePerType.sv_counts
-    Array[File?] sv_count_plots = PlotSVCountsWithCutoff.svcount_distrib_plots
+    Array[File] sv_counts = select_all(CountSVsPerSamplePerType.sv_counts)
+    Array[File] sv_count_plots = select_all(PlotSVCountsWithCutoff.svcount_distrib_plots)
     File outlier_samples_preview = CatOutliersPreview.outliers_preview_file
   }
 }