initial draft of separated aws

Author: dsweber2

Makefile

@@ -7,3 +7,6 @@ run:
 
 run-nohup:
 	nohup Rscript run.R &
+
+sync:
+	Rscript sync.R

NAMESPACE

@@ -27,6 +27,7 @@ export(make_shared_ensembles)
 export(make_shared_grids)
 export(make_target_ensemble_grid)
 export(make_target_param_grid)
+export(manage_S3_forecast_cache)
 export(overprediction)
 export(perform_sanity_checks)
 export(read_external_predictions_data)
@@ -39,6 +40,8 @@ export(slide_forecaster)
 export(underprediction)
 export(weighted_interval_score)
 importFrom(assertthat,assert_that)
+importFrom(aws.s3,get_bucket)
+importFrom(aws.s3,s3sync)
 importFrom(cli,cli_abort)
 importFrom(cli,hash_animal)
 importFrom(dplyr,across)
@@ -82,6 +85,7 @@ importFrom(epipredict,step_population_scaling)
 importFrom(epipredict,step_training_window)
 importFrom(epiprocess,as_epi_df)
 importFrom(epiprocess,epix_slide)
+importFrom(here,here)
 importFrom(magrittr,"%<>%")
 importFrom(magrittr,"%>%")
 importFrom(purrr,imap)

R/manage_S3.R

@@ -0,0 +1,47 @@
+#' Manage S3 cache
+#' @param rel_cache_dir The relative path to the cache directory, e.g.
+#' "data-processed/2021-09-01". Default is `"{tar_project}/objects"`
+#' @param bucket_name The name of the S3 bucket to sync.
+#' @param direction Set 'download' to download files, 'upload' to upload files,
+#'   and `sync` to do both
+#' @param verbose Set to TRUE to print the files being synced.
+#' @param prefix specify the prefix for `s3sync`, which filters down which files
+#'   to sync to those starting with `prefix`
+#'
+#' @importFrom aws.s3 s3sync get_bucket
+#' @importFrom here here
+#' @export
+manage_S3_forecast_cache <- function(rel_cache_dir = NULL,
+                                     bucket_name = "forecasting-team-data",
+                                     direction = "sync",
+                                     verbose = FALSE,
+                                     prefix = Sys.getenv("AWS_S3_PREFIX", "exploration"),
+                                     tar_project = Sys.getenv("TAR_PROJECT", "exploration")) {
+  if (is.null(rel_cache_dir)) {
+    cache_path <- tar_project
+  } else {
+    cache_path <- here(rel_cache_dir)
+  }
+  if (!dir.exists(cache_path)) dir.create(cache_path)
+
+  prefix <- paste0(prefix, "/", tar_project, "/")
+  s3b <- get_bucket(bucket_name, prefix = prefix)
+  if (direction == "sync") {
+    if (verbose) {
+      s3sync(cache_path, s3b, prefix = prefix)
+    } else {
+      sink("/dev/null")
+      s3sync(cache_path, s3b, prefix = prefix, verbose = FALSE)
+      sink()
+    }
+  } else {
+    if (verbose) {
+      s3sync(cache_path, s3b, prefix = prefix, direction = direction)
+    } else {
+      sink("/dev/null")
+      s3sync(cache_path, s3b, prefix = prefix, direction = direction, verbose = FALSE)
+      sink()
+    }
+  }
+  return(TRUE)
+}

README.md

@@ -16,9 +16,11 @@ DEBUG_MODE=true
 USE_SHINY=false
 TAR_PROJECT=covid_hosp_explore
 EXTERNAL_SCORES_PATH=
-USE_AWS_S3=false
 AWS_S3_PREFIX=2023/exploration
 
+# Pull from the bucket
+make sync
+
 # Run the pipeline wrapper run.R.
 make run
 ```
@@ -28,7 +30,6 @@ make run
 -   `USE_SHINY` controls whether we start a Shiny server after producing the targets.
 -   `TAR_PROJECT` controls which `targets` project is run by `run.R`.
 -   `EXTERNAL_SCORES_PATH` controls where external scores are loaded from. If not set, external scores are not used.
--   `USE_AWS_S3` controls whether we use AWS S3 to store the cache.
 -   `AWS_S3_PREFIX` controls the prefix to use in the AWS S3 bucket (a prefix is a pseudo-directory in a bucket).
 
 ## Development

extras/targets-common.R

@@ -50,23 +50,6 @@ tar_option_set(
   )
 )
 
-use_aws_s3 <- as.logical(Sys.getenv("USE_AWS_S3", FALSE))
-tar_project <- Sys.getenv("TAR_PROJECT", "covid_hosp_explore")
-aws_s3_prefix <- Sys.getenv("AWS_S3_PREFIX", "exploration")
-aws_s3_prefix <- paste0(aws_s3_prefix, "/", tar_project)
-if (use_aws_s3) {
-  tar_option_set(
-    repository = "aws",
-    resources = tar_resources(
-      aws = tar_resources_aws(
-        bucket = "forecasting-team-data",
-        prefix = aws_s3_prefix,
-        region = "us-east-1"
-      )
-    )
-  )
-}
-
 linreg <- parsnip::linear_reg()
 quantreg <- epipredict::quantile_reg()
 ONE_AHEAD_FORECAST_NAME <- "forecast_by_ahead"