Option to provide credentials JSON to access CRAMs & fix bug in Whamg (…

…#333)

wdl/CramToBam.wdl

@@ -1,15 +1,20 @@
 version 1.0
 
 import "Structs.wdl"
+import "Utils.wdl" as util
 
 workflow CramToBam {
   input {
     File cram_file
     File reference_fasta
     File? reference_index
+    String? service_account_json  # Provide path to service account credentials JSON if required to access CRAM file. Not supported for requester pays CRAMs, BAM access, or revising bases
     String samtools_cloud_docker
+    String? cloud_sdk_docker
     Boolean requester_pays = false
+    Int? localize_cram_disk_size
     RuntimeAttr? runtime_attr_override
+    RuntimeAttr? runtime_attr_localize_cram
   }
 
   parameter_meta {
@@ -22,7 +27,18 @@ workflow CramToBam {
       email: "[email protected]"
   }
 
-  if (requester_pays) {
+  if (defined(service_account_json)) {
+    call util.LocalizeCloudFileWithCredentials as LocalizeCram {
+      input:
+        cloud_file_path = cram_file,
+        service_account_json = select_first([service_account_json]),
+        disk_size = select_first([localize_cram_disk_size, 50]),
+        cloud_sdk_docker = select_first([cloud_sdk_docker]),
+        runtime_attr_override = runtime_attr_localize_cram
+    }
+  }
+
+  if (requester_pays && !defined(service_account_json)) {
     call RunCramToBamRequesterPays {
       input:
         cram_file = cram_file,
@@ -35,7 +51,7 @@ workflow CramToBam {
   if (!requester_pays) {
     call RunCramToBam {
       input:
-        cram_file = cram_file,
+        cram_file = select_first([LocalizeCram.output_file, cram_file]),
         reference_fasta = reference_fasta,
         reference_index = reference_index,
         samtools_cloud_docker = samtools_cloud_docker,

wdl/GatherSampleEvidence.wdl

@@ -22,6 +22,10 @@ workflow GatherSampleEvidence {
     # Use only for crams in requester pays buckets
     Boolean requester_pays_crams = false
 
+    # Provide path to service account credentials JSON if required to access CRAM file. 
+    # Not supported for requester pays CRAMs, BAM access, or revising bases
+    String? service_account_json
+
     # Use to revise Y, R, W, S, K, M, D, H, V, B, X bases in BAM to N. Use only if providing a CRAM file as input 
     # May be more expensive - use only if necessary
     Boolean revise_base_cram_to_bam = false
@@ -114,6 +118,7 @@ workflow GatherSampleEvidence {
     RuntimeAttr? runtime_attr_wham_include_list
     RuntimeAttr? runtime_attr_ReviseBaseInBam
     RuntimeAttr? runtime_attr_ConcatBam
+    RuntimeAttr? runtime_attr_localize_cram
 
     # Never assign these values! (workaround until None type is implemented)
     Float? NONE_FLOAT_
@@ -140,7 +145,10 @@ workflow GatherSampleEvidence {
           reference_fasta = reference_fasta,
           reference_index = reference_index,
           requester_pays = requester_pays_crams,
+          service_account_json = service_account_json,
           samtools_cloud_docker = samtools_cloud_docker,
+          cloud_sdk_docker = cloud_sdk_docker,
+          runtime_attr_localize_cram = runtime_attr_localize_cram,
           runtime_attr_override = runtime_attr_cram_to_bam
       }
     }

wdl/Utils.wdl

@@ -313,3 +313,47 @@ task SubsetPedFile {
     maxRetries: select_first([runtime_attr.max_retries, default_attr.max_retries])
   }
 }
+
+task LocalizeCloudFileWithCredentials {
+  input {
+    String cloud_file_path
+    String service_account_json
+    Int disk_size
+    String cloud_sdk_docker
+    RuntimeAttr? runtime_attr_override
+  }
+
+  RuntimeAttr default_attr = object {
+    cpu_cores: 1,
+    mem_gb: 0.9,
+    disk_gb: disk_size,
+    boot_disk_gb: 10,
+    preemptible_tries: 3,
+    max_retries: 1
+  }
+  RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
+
+  command {
+    set -euo pipefail
+
+    gsutil cp '~{service_account_json}' local.service_account.json
+    gcloud auth activate-service-account --key-file='local.service_account.json'
+
+    gsutil cp '~{cloud_file_path}' .
+  }
+
+  output {
+    File output_file = basename(cloud_file_path)
+  }
+
+  runtime {
+    cpu: select_first([runtime_attr.cpu_cores, default_attr.cpu_cores])
+    memory: select_first([runtime_attr.mem_gb, default_attr.mem_gb]) + " GiB"
+    disks: "local-disk " + select_first([runtime_attr.disk_gb, default_attr.disk_gb]) + " HDD"
+    bootDiskSizeGb: select_first([runtime_attr.boot_disk_gb, default_attr.boot_disk_gb])
+    docker: cloud_sdk_docker
+    preemptible: select_first([runtime_attr.preemptible_tries, default_attr.preemptible_tries])
+    maxRetries: select_first([runtime_attr.max_retries, default_attr.max_retries])
+  }
+}
+

wdl/Whamg.wdl

@@ -158,7 +158,6 @@ task RunWhamg {
     set -euo pipefail
     # print some info that may be useful for debugging
     df -h
-    cat /sys/fs/cgroup/memory/memory.stat
     echo "whamg $(./whamg 2>&1 >/dev/null | grep Version)"
 
     # ensure that index files are present in appropriate locations
@@ -295,7 +294,6 @@ task RunWhamgIncludelist {
     set -euo pipefail
     # print some info that may be useful for debugging
     df -h
-    cat /sys/fs/cgroup/memory/memory.stat
     echo "whamg $(./whamg 2>&1 >/dev/null | grep Version)"
 
     # ensure that index files are present in appropriate locations