Merge pull request #69 from cmu-delphi/ds/aws-s3

feat: add s3 sync option for targets

Author: dshemetov

README.md

@@ -16,6 +16,8 @@ DEBUG_MODE=true
 USE_SHINY=false
 TAR_PROJECT=covid_hosp_explore
 EXTERNAL_SCORES_PATH=
+USE_AWS_S3=false
+AWS_S3_PREFIX=
 
 # Run the pipeline wrapper run.R.
 make run
@@ -26,6 +28,8 @@ make run
 -   `USE_SHINY` controls whether we start a Shiny server after producing the targets.
 -   `TAR_PROJECT` controls which `targets` project is run by `run.R`.
 -   `EXTERNAL_SCORES_PATH` controls where external scores are loaded from. If not set, external scores are not used.
+-   `USE_AWS_S3` controls whether we use AWS S3 to store the cache.
+-   `AWS_S3_PREFIX` controls the prefix to use in the AWS S3 bucket (a prefix is a pseudo-directory in a bucket).
 
 ## Development
 

extras/extra-packages.R

@@ -3,6 +3,7 @@ library(fs)
 library(glue)
 library(languageserver)
 library(pak)
+library(paws.storage)
 library(quantreg)
 library(renv)
 library(rspm)

renv.lock

@@ -799,7 +799,7 @@
       "RemoteUsername": "cmu-delphi",
       "RemoteRepo": "epidatr",
       "RemoteRef": "dev",
-      "RemoteSha": "ee952c518ac0cf7cc25083b6c54a0b284d2c3448",
+      "RemoteSha": "83e986b90ff48a972a2de972c184ec06bcce2113",
       "Requirements": [
         "MMWRweek",
         "R",
@@ -816,7 +816,7 @@
         "tibble",
         "xml2"
       ],
-      "Hash": "c628a40210a598e5c165e4a31c7d781b"
+      "Hash": "a0cd96909bf0dc5d470f5ed08fb81ff7"
     },
     "epipredict": {
       "Package": "epipredict",
@@ -1772,6 +1772,35 @@
       ],
       "Hash": "cbb49c720a5e4bf1fe36a870b07d1a0c"
     },
+    "paws.common": {
+      "Package": "paws.common",
+      "Version": "0.6.4",
+      "Source": "Repository",
+      "Repository": "RSPM",
+      "Requirements": [
+        "Rcpp",
+        "base64enc",
+        "curl",
+        "digest",
+        "httr",
+        "jsonlite",
+        "methods",
+        "stats",
+        "utils",
+        "xml2"
+      ],
+      "Hash": "fcc0e7509d0c9da0874b5d3a7d8ea904"
+    },
+    "paws.storage": {
+      "Package": "paws.storage",
+      "Version": "0.4.0",
+      "Source": "Repository",
+      "Repository": "RSPM",
+      "Requirements": [
+        "paws.common"
+      ],
+      "Hash": "8aea3bee1ea38991f23aea27583b93ef"
+    },
     "pillar": {
       "Package": "pillar",
       "Version": "1.9.0",
@@ -2216,14 +2245,14 @@
     },
     "rlang": {
       "Package": "rlang",
-      "Version": "1.1.1",
+      "Version": "1.1.2",
       "Source": "Repository",
-      "Repository": "CRAN",
+      "Repository": "RSPM",
       "Requirements": [
         "R",
         "utils"
       ],
-      "Hash": "a85c767b55f0bf9b7ad16c6d7baee5bb"
+      "Hash": "50a6dbdc522936ca35afc5e2082ea91b"
     },
     "rmarkdown": {
       "Package": "rmarkdown",

run.R

@@ -10,6 +10,7 @@
 #   # Example fetching external scores from the forecasting bucket
 #   library(aws.s3)
 #
+#   # We recommend setting these in your user config files such as ~/.zprofile.
 #   Sys.setenv(
 #     AWS_ACCESS_KEY_ID = "",
 #     AWS_SECRET_ACCESS_KEY = ""
@@ -25,17 +26,23 @@
 #   # Save to disk
 #   saveRDS(scorecards, "exploration-scorecards-2023-10-04.RDS")
 
-print("Reading environment variables (TAR_PROJECT, EXTERNAL_SCORES_PATH, DEBUG_MODE, USE_SHINY)...")
 tar_project <- Sys.getenv("TAR_PROJECT", "covid_hosp_explore")
 external_scores_path <- Sys.getenv("EXTERNAL_SCORES_PATH", "")
 debug_mode <- as.logical(Sys.getenv("DEBUG_MODE", TRUE))
 use_shiny <- as.logical(Sys.getenv("USE_SHINY", FALSE))
-
-cat("Using project: ", tar_project, "\n")
-if (external_scores_path != "") cat("Using external scores from ", external_scores_path, "\n")
-if (debug_mode) cat("Debug mode is on.")
-if (use_shiny) cat("Running shiny server after results.")
-
+use_aws_s3 <- as.logical(Sys.getenv("USE_AWS_S3", FALSE))
+aws_s3_prefix <- Sys.getenv("AWS_S3_PREFIX", "exploration")
+cli::cli_inform(
+  c(
+    "i" = "Reading environment variables...",
+    "*" = "TAR_PROJECT = {tar_project}",
+    "*" = "EXTERNAL_SCORES_PATH = {external_scores_path}",
+    "*" = "DEBUG_MODE = {debug_mode}",
+    "*" = "USE_SHINY = {use_shiny}",
+    "*" = "USE_AWS_S3 = {use_aws_s3}",
+    "*" = "AWS_S3_PREFIX = {aws_s3_prefix}"
+  )
+)
 
 suppressPackageStartupMessages({
   library(targets)
@@ -46,6 +53,18 @@ suppressPackageStartupMessages({
 store_dir <- tar_path_store()
 if (!dir.exists(store_dir)) dir.create(store_dir)
 
+if (use_aws_s3) {
+  tar_option_set(
+    repository = "aws",
+    resources = tar_resources(
+      aws = tar_resources_aws(
+        bucket = "forecasting-team-data",
+        prefix = aws_s3_prefix
+      )
+    )
+  )
+}
+
 tar_manifest()
 if (debug_mode) {
   tar_make(callr_function = NULL)