finish moving to getters, run through partitions, filter out lake and…

… miscellaneous sites. Closes USGS-R#18 and USGS-R#19

1_wqp_pull/inout/wqp_pull_partitions.feather.ind

@@ -1,2 +1,2 @@
-hash: 7b4b0b0d6dcb3d364e32fdda5c390c17
+hash: 36b12b88022da6bdea48d05dd4e28d19
 

2_nwis_pull.yml

@@ -20,8 +20,11 @@ sources:
 targets:
   2_nwis_pull:
     depends:
-      - 2_nwis_pull/out/nwis_dv_data.rds.ind
-      - 2_nwis_pull/out/nwis_uv_data.rds.ind
+      - 2_nwis_pull/inout/nwis_dv_inventory_summary.csv
+      - 2_nwis_pull/inout/nwis_uv_inventory_summary.csv
+      - 2_nwis_pull/out/nwis_dv_summary.csv
+      - 2_nwis_pull/out/nwis_uv_summary.csv
+
 
 # -- get a CONUS inventory of available data for download -- #
 
@@ -83,16 +86,22 @@ targets:
 
   # -- partition data -- #
   # dv
+
+  site_type_keep:
+    command: c(I(c('ST', 'ST-CA', 'ST-DCH', 'ST-TS', 'SP')))
+
   2_nwis_pull/inout/nwis_dv_partition.feather.ind:
     command: partition_inventory(inventory_ind = '2_nwis_pull/inout/nwis_dv_inventory.feather.ind', 
       nwis_pull_size, 
+      site_types = site_type_keep,
       partitions_ind = target_name,
       pull_date = nwis_dv_pull_date) 
 
   #uv
   2_nwis_pull/inout/nwis_uv_partition.feather.ind:
     command: partition_inventory(inventory_ind = '2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind', 
-      nwis_pull_size, 
+      nwis_pull_size,
+      site_types = site_type_keep,
       partitions_ind = target_name,
       pull_date = nwis_uv_pull_date) 
 

2_nwis_pull/inout/nwis_compare_services.feather.ind

@@ -1,2 +1,2 @@
-hash: 59733420f0f8843344d07f38054f7a1b
+hash: 75f7ef2f0173b81e1cb824ae99b3abad
 

2_nwis_pull/inout/nwis_dv_inventory.feather.ind

@@ -1,2 +1,2 @@
-hash: 9040f9f5b1662ecf237c94ec3f0a7f45
+hash: 33d6f75db2ffeae7e8b611f503fd45a9
 

2_nwis_pull/inout/nwis_dv_partition.feather.ind

@@ -1,2 +1,2 @@
-hash: b62cd04c5d4baa79153477a2d7dac31a
+hash: 8019ab09d467ac4357004f358a8a671a
 

2_nwis_pull/inout/nwis_uv_inventory.feather.ind

@@ -1,2 +1,2 @@
-hash: 74dce1745d0c9f605b7ce76519e4f010
+hash: 8c6c537a90b96585775ef0c51ab2a029
 

2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind

@@ -1,2 +1,2 @@
-hash: c48447f327baaee276e76d367b49ec1f
+hash: bd79c019a940e022dd31c2a325a431fe
 

2_nwis_pull/inout/nwis_uv_partition.feather.ind

@@ -1,2 +1,2 @@
-hash: 5941dff7fa93f248da209c8a04ab30ed
+hash: 8679e720a4396131a2a41646baa12ddc
 

2_nwis_pull/src/compare_nwis_services.R

@@ -1,8 +1,8 @@
 # compare NWIS dv and uv services
 
 compare_services <- function(dv_ind, uv_ind, compare_ind) {
-  uv <- feather::read_feather(sc_retrieve(uv_ind))
-  dv <- feather::read_feather(sc_retrieve(dv_ind))
+  uv <- feather::read_feather(sc_retrieve(uv_ind, remake_file = 'getters.yml'))
+  dv <- feather::read_feather(sc_retrieve(dv_ind, remake_file = 'getters.yml'))
 
   uv <- uv %>%
     select(site_no, uv_begin_date = begin_date, uv_end_date = end_date, uv_count = count_nu) %>%
@@ -23,8 +23,8 @@ compare_services <- function(dv_ind, uv_ind, compare_ind) {
 }
 
 reduce_redundancy <- function(uv_inv_ind, compare_ind, out_ind) {
-  compare <- feather::read_feather(sc_retrieve(compare_ind))
-  uv_inv <- feather::read_feather(sc_retrieve(uv_inv_ind))
+  compare <- feather::read_feather(sc_retrieve(compare_ind, remake_file = 'getters.yml'))
+  uv_inv <- feather::read_feather(sc_retrieve(uv_inv_ind, remake_file = 'getters.yml'))
 
   uv_pull_sites <- filter(compare, is.na(dv_begin_date)) %>%
     select(site_no) %>%

2_nwis_pull/src/inventory_nwis.R

@@ -24,7 +24,7 @@ inventory_nwis <- function(inv_ind, nwis_pull_params, service) {
 
 summarize_nwis_inventory <- function(inv_ind, out_file) {
 
-  nwis_inventory <- feather::read_feather(sc_retrieve(inv_ind))
+  nwis_inventory <- feather::read_feather(sc_retrieve(inv_ind, remake_file = 'getters.yml'))
 
   all <- data.frame(n_sites = nrow(nwis_inventory), 
                     n_records = sum(nwis_inventory$count_nu), 
@@ -37,7 +37,7 @@ summarize_nwis_inventory <- function(inv_ind, out_file) {
 }
 
 summarize_nwis_data <- function(data_ind, out_file) {
-  nwis_data <- readRDS(sc_retrieve(data_ind))
+  nwis_data <- readRDS(sc_retrieve(data_ind, remake_file = 'getters.yml'))
 
   summary <- data.frame(n_obs = nrow(nwis_data),
                         n_sites = length(unique(nwis_data$site_no)))

2_nwis_pull/src/nwis_partition.R

@@ -1,6 +1,6 @@
-partition_inventory <- function(inventory_ind, nwis_pull_size, partitions_ind, pull_date) {
+partition_inventory <- function(inventory_ind, nwis_pull_size, site_types, partitions_ind, pull_date) {
 
-  inventory <- feather::read_feather(scipiper::sc_retrieve(inventory_ind,remake_file = '2_nwis_pull.yml'))
+  inventory <- feather::read_feather(scipiper::sc_retrieve(inventory_ind, remake_file = 'getters.yml'))
 
   # uv data count number is the number of days between the min and max observation days
   # assume that each day has 15 minute data, which is 96 obs/day
@@ -12,6 +12,7 @@ partition_inventory <- function(inventory_ind, nwis_pull_size, partitions_ind, p
   # first, get rid of duplicate sites from whatNWISdata call
   # do not need to pull sites twice
   atomic_groups <- inventory %>%
+    filter(site_tp_cd %in% site_types) %>%
     group_by(site_no) %>%
     slice(which.max(count_nu)) %>%
     ungroup() %>%

2_nwis_pull/src/nwis_pull.R

@@ -9,7 +9,7 @@ plan_nwis_pull <- function(partitions_ind, service) {
     out='2_nwis_pull/out',
     log='2_nwis_pull/log')
 
-  partitions <- feather::read_feather(scipiper::sc_retrieve(partitions_ind))
+  partitions <- feather::read_feather(scipiper::sc_retrieve(partitions_ind, remake_file = 'getters.yml'))
 
   # after all wanted data have been pulled, this function will be called but
   # doesn't need to create anything much, so just return NULL
@@ -99,7 +99,7 @@ combine_nwis_data <- function(ind_file, ...){
 # PullTask 
 filter_partitions <- function(partitions_ind, pull_task) {
 
-  partitions <- feather::read_feather(sc_retrieve(partitions_ind))
+  partitions <- feather::read_feather(sc_retrieve(partitions_ind, remake_file = 'getters.yml'))
 
   these_partitions <- dplyr::filter(partitions, PullTask==pull_task, count_nu > 0)
 

5_data_munge.yml

@@ -31,38 +31,26 @@ targets:
       in_ind = '1_wqp_pull/out/wqp_data.rds.ind',
       min_value, max_value, max_daily_range, 
       out_ind = target_name)
-
-  5_data_munge/out/wqp_daily_depths.rds:
-    command: gd_get('5_data_munge/out/wqp_daily_depths.rds.ind')
 
   5_data_munge/out/wqp_daily_nodepths.rds.ind:
     command: munge_wqp_withoutdepths(
       in_ind = '1_wqp_pull/out/wqp_data.rds.ind',
       min_value, max_value, max_daily_range, 
       out_ind = target_name)
-
-  5_data_munge/out/wqp_daily_nodepths.rds:
-    command: gd_get('5_data_munge/out/wqp_daily_nodepths.rds.ind')
 
   # NWIS uv data #
   5_data_munge/out/nwis_uv_daily.rds.ind:
     command: munge_nwis_uv(
       in_ind = '2_nwis_pull/out/nwis_uv_data.rds.ind',
       min_value, max_value, max_daily_range, 
       out_ind = target_name)
-
-  5_data_munge/out/nwis_uv_daily.rds:
-    command: gd_get('5_data_munge/out/nwis_uv_daily.rds.ind')
 
   # NWIS dv data #
   5_data_munge/out/nwis_dv_daily.rds.ind:
     command: munge_nwis_dv(
       in_ind = '2_nwis_pull/out/nwis_dv_data.rds.ind',
       min_value, max_value, max_daily_range, 
       out_ind = target_name)
-
-  5_data_munge/out/nwis_dv_daily.rds:
-    command: gd_get('5_data_munge/out/nwis_dv_daily.rds.ind')
 
   # EcoSHEDS data
   5_data_munge/out/ecosheds_munged.rds.ind:
@@ -78,17 +66,11 @@ targets:
       wqp_ind = '5_data_munge/out/wqp_daily_nodepths.rds.ind',
       ecosheds_ind = '5_data_munge/out/ecosheds_munged.rds.ind',
       out_ind = target_name)
-
-  5_data_munge/out/daily_temperatures.rds:
-    command: gd_get('5_data_munge/out/daily_temperatures.rds.ind')
-
+
   5_data_munge/out/all_sites.rds.ind:
     command: combine_all_sites(
       out_ind = target_name,
       nwis_dv_sites_ind = '2_nwis_pull/inout/nwis_dv_inventory.feather.ind',
       nwis_uv_sites_ind = '2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind',
       ecosheds_sites_ind = '4_other_sources/out/ecosheds_sites_ind',
       wqp_sites_ind = '1_wqp_pull/inout/wqp_inventory.feather.ind')
-
-  5_data_munge/out/all_sites.rds:
-    command: gd_get('5_data_munge/out/all_sites.rds.ind')

build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml

@@ -1,14 +1,15 @@
 version: 0.3.0
 name: 1_wqp_pull/inout/wqp_pull_partitions.feather.ind
 type: file
-hash: 952142a2367540f72e08359cdf8b08b0
-time: 2020-01-08 16:37:06 UTC
+hash: 12a87b82c3d90bee280b0b8167a83e29
+time: 2020-07-24 16:27:15 UTC
 depends:
   wqp_pull_parameters: 61e80f749c74948bf95bb67cb07412b0
-  1_wqp_pull/inout/wqp_inventory.feather.ind: 25f8549426744ab16e779fee70c761c4
+  1_wqp_pull/inout/wqp_inventory.feather.ind: fadf3bf3c179f5b48e58503abe0e12ad
   wqp_partition_cfg: dc9f1a6787144e2e96aafb48b1ee67ce
+  wqp_pull_date: 1fbc5fcb5a023a50317972d9e65e1d54
 fixed: f1ee901bf2a86f4bac0c08cf4480a175
 code:
   functions:
-    partition_wqp_inventory: b9fab76a65793640fa18115442ea01c1
+    partition_wqp_inventory: 70d1ffc68d6179c10c49fa4994b1afa4
 

build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9pbnZlbnRvcnkuZmVhdGhlci5pbmQ.yml

@@ -1,10 +1,11 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_uv_inventory.feather.ind
 type: file
-hash: 56e108be5e959a84ba30410f16dc156e
-time: 2020-01-28 15:37:52 UTC
+hash: 5913d3404169fc4027fc065f14d456d4
+time: 2020-07-24 21:24:46 UTC
 depends:
-  nwis_pull_parameters: 66f6e5244eaa5c902b135f2d2a23f1c9
+  nwis_pull_parameters: 6b3638cd4c2d2b5f9a67606053e9e271
+  nwis_uv_pull_date: 4c2c8b55aa9e326608d1337eea7433a0
 fixed: 93ef43b432abb3090878b26e42599931
 code:
   functions:

build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9pbnZlbnRvcnlfcmVkdWNlZC5mZWF0aGVyLmluZA.yml

@@ -1,13 +1,13 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind
 type: file
-hash: 1af0f70101f0176adfc9643d1e1b8cb2
-time: 2020-06-30 21:05:05 UTC
+hash: af13229385810bef9f4dd8a87044c544
+time: 2020-07-24 21:37:15 UTC
 depends:
-  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: d516e14fb29bb1cb34035a37f299823e
-  2_nwis_pull/inout/nwis_compare_services.feather.ind: f03409f8074adc4bf421facb3ff5bb6c
+  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: 5913d3404169fc4027fc065f14d456d4
+  2_nwis_pull/inout/nwis_compare_services.feather.ind: 430f4ec0e02df97f1748d9da964580c7
 fixed: 87462faeb4ae7586edd11b9687c2273a
 code:
   functions:
-    reduce_redundancy: f90ae0e64a66d39381cf27aafb01202b
+    reduce_redundancy: 52d3da9662110794bf56689618cd9068
 

build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml

@@ -1,13 +1,15 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_uv_partition.feather.ind
 type: file
-hash: feaf591059b86dc78223cd1de9f7f797
-time: 2020-01-28 16:42:17 UTC
+hash: e1ad83be21745aff3ca25da4c64871e7
+time: 2020-07-26 20:58:52 UTC
 depends:
-  2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind: 1af0f70101f0176adfc9643d1e1b8cb2
+  2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind: af13229385810bef9f4dd8a87044c544
   nwis_pull_size: bf6a897122707e205d7ad26fc0132080
+  site_type_keep: 1dda0c69f59b81fcad3048999043a0a9
+  nwis_uv_pull_date: 4c2c8b55aa9e326608d1337eea7433a0
 fixed: ac3fcec4b396ccf004113a8dc5e03171
 code:
   functions:
-    partition_inventory: 59c05ace5764b063b7856cb325bc46cc
+    partition_inventory: 0609333f17f066e0a4a7b7df80a8ce41
 

build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19jb21wYXJlX3NlcnZpY2VzLmZlYXRoZXIuaW5k.yml

@@ -1,13 +1,13 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_compare_services.feather.ind
 type: file
-hash: f03409f8074adc4bf421facb3ff5bb6c
-time: 2020-06-30 21:04:52 UTC
+hash: 430f4ec0e02df97f1748d9da964580c7
+time: 2020-07-24 21:25:06 UTC
 depends:
-  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: a7d1a763912f7f7bbbe36d6fab32c15f
-  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: d516e14fb29bb1cb34035a37f299823e
+  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: ed24501764639dc862c9559d04795226
+  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: 5913d3404169fc4027fc065f14d456d4
 fixed: da0ba4d054d65f999fa79e7d3e7fc0a5
 code:
   functions:
-    compare_services: 5b3d393df8f6f09891f31c6d6ceff9bb
+    compare_services: 130e4dbb104c76eb2ff193e8f9299785
 

build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19kdl9pbnZlbnRvcnkuZmVhdGhlci5pbmQ.yml

@@ -1,10 +1,11 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_dv_inventory.feather.ind
 type: file
-hash: 7ce770b5124fd9c9d9e947dfb9c8c07c
-time: 2020-01-28 15:18:15 UTC
+hash: ed24501764639dc862c9559d04795226
+time: 2020-07-24 21:07:08 UTC
 depends:
-  nwis_pull_parameters: 66f6e5244eaa5c902b135f2d2a23f1c9
+  nwis_pull_parameters: 6b3638cd4c2d2b5f9a67606053e9e271
+  nwis_dv_pull_date: 4c2c8b55aa9e326608d1337eea7433a0
 fixed: 9d698da59c8ce8b6b3fa4523906ea38d
 code:
   functions:

build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19kdl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml

@@ -1,13 +1,15 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_dv_partition.feather.ind
 type: file
-hash: 3e2dd8898901ae6927e1c6afd99767da
-time: 2020-07-01 16:49:14 UTC
+hash: c993f5b96eb326841f4cb850a4bdb1b2
+time: 2020-07-26 20:57:33 UTC
 depends:
-  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: a7d1a763912f7f7bbbe36d6fab32c15f
+  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: ed24501764639dc862c9559d04795226
   nwis_pull_size: bf6a897122707e205d7ad26fc0132080
+  site_type_keep: 1dda0c69f59b81fcad3048999043a0a9
+  nwis_dv_pull_date: 4c2c8b55aa9e326608d1337eea7433a0
 fixed: 4679fb1ac3904ee445bbc34f4a62eb19
 code:
   functions:
-    partition_inventory: 59c05ace5764b063b7856cb325bc46cc
+    partition_inventory: 0609333f17f066e0a4a7b7df80a8ce41
 

getters.yml

@@ -63,3 +63,22 @@ targets:
   5_data_munge/out/ecosheds_munged.rds:
     command: gd_get('5_data_munge/out/ecosheds_munged.rds.ind')
 
+
+  5_data_munge/out/wqp_daily_depths.rds:
+    command: gd_get('5_data_munge/out/wqp_daily_depths.rds.ind')
+
+
+  5_data_munge/out/wqp_daily_nodepths.rds:
+    command: gd_get('5_data_munge/out/wqp_daily_nodepths.rds.ind')
+
+  5_data_munge/out/nwis_uv_daily.rds:
+    command: gd_get('5_data_munge/out/nwis_uv_daily.rds.ind')
+
+  5_data_munge/out/nwis_dv_daily.rds:
+    command: gd_get('5_data_munge/out/nwis_dv_daily.rds.ind')
+
+  5_data_munge/out/daily_temperatures.rds:
+    command: gd_get('5_data_munge/out/daily_temperatures.rds.ind')
+
+  5_data_munge/out/all_sites.rds:
+    command: gd_get('5_data_munge/out/all_sites.rds.ind')