gcp workspaces

Author: wlandau

Diff for: R/class_aws.R

@@ -177,8 +177,9 @@ store_delete_objects.tar_aws <- function(store, meta, batch_size, verbose) {
       store_aws_region(.x),
       store_aws_endpoint(.x),
       sep = "|"
-    )
+    ) %||% NA_character_
   )
+  meta <- meta[!is.na(meta$bucket_group), ]
   for (group in unique(meta$bucket_group)) {
     subset <- meta[meta$bucket_group == group,, drop = FALSE] # nolint
     example_path <- subset$path[[1L]]

Diff for: R/class_database_gcp.R

@@ -65,6 +65,28 @@ database_gcp_class <- R6::R6Class(
       )
       invisible()
     },
+    download_workspace = function(name, store, verbose = TRUE) {
+      path <- path_workspace(store, name)
+      key <- path_workspace(dirname(dirname(self$key)), name)
+      gcp <- self$resources$gcp
+      if (verbose) {
+        tar_print(
+          "Downloading gcp workspace file ",
+          key,
+          " to local file ",
+          path
+        )
+      }
+      dir_create(dirname(path))
+      gcp_gcs_download(
+        file = path,
+        key = key,
+        bucket = gcp$bucket,
+        max_tries = gcp$max_tries %|||% 5L,
+        verbose = gcp$verbose %|||% TRUE
+      )
+      invisible()
+    },
     upload = function(verbose = TRUE) {
       if (verbose) {
         tar_print(
@@ -92,6 +114,22 @@ database_gcp_class <- R6::R6Class(
       )
       invisible()
     },
+    upload_workspace = function(target, meta, reporter) {
+      name <- target_get_name(target)
+      path <- path_workspace(meta$store, name)
+      key <- path_workspace(dirname(dirname(self$key)), name)
+      gcp <- self$resources$gcp
+      gcp_gcs_upload(
+        file = path,
+        key = key,
+        bucket = gcp$bucket,
+        predefined_acl = gcp$predefined_acl %|||% "private",
+        max_tries = gcp$max_tries %|||% 5L,
+        verbose = gcp$verbose %|||% TRUE
+      )
+      reporter$report_workspace_upload(target)
+      invisible()
+    },
     head = function() {
       gcp <- self$resources$gcp
       head <- gcp_gcs_head(
@@ -118,7 +156,29 @@ database_gcp_class <- R6::R6Class(
         max_tries = gcp$max_tries %|||% 5L,
         verbose = gcp$verbose %|||% TRUE
       )
+    },
+    delete_cloud_workspaces = function() {
+      prefix <- dirname(path_workspace(dirname(dirname(self$key)), "x"))
+      gcp <- self$resources$gcp
+      keys <- names(
+        gcp_gcs_list_md5s(
+          prefix = prefix,
+          bucket = gcp$bucket,
+          verbose = FALSE,
+          max_tries = gcp$max_tries %|||% 5L
+        )
+      )
+      for (key in keys) {
+        gcp_gcs_delete(
+          key = key,
+          bucket = gcp$bucket,
+          verbose = FALSE,
+          max_tries = gcp$max_tries %|||% 5L
+        )
+      }
+      invisible()
     }
+    
   )
 )
 # nocov end

Diff for: R/class_gcp.R

@@ -132,11 +132,14 @@ store_delete_objects.tar_gcp <- function(store, meta, batch_size, verbose) {
   gcp <- store$resources$gcp
   meta$bucket_group <- map_chr(
     x = meta$path,
-    f = ~paste(
-      store_gcp_bucket(.x),
-      sep = "|"
-    )
+    f = function(x) {
+      paste(
+        store_gcp_bucket(x),
+        sep = "|"
+      ) %||% NA_character_
+    }
   )
+  meta <- meta[!is.na(meta$bucket_group), ]
   for (group in unique(meta$bucket_group)) {
     subset <- meta[meta$bucket_group == group,, drop = FALSE] # nolint
     for (index in seq_len(nrow(subset))) {

Diff for: targets.Rproj

@@ -1,5 +1,4 @@
 Version: 1.0
-ProjectId: 509dc227-2bd1-454b-86f9-e6d90611f2e1
 
 RestoreWorkspace: Default
 SaveWorkspace: Default

Diff for: tests/aws/test-aws-workspaces.R

@@ -2,7 +2,6 @@
 # Verify all `targets` buckets are deleted afterwards.
 tar_test("aws workspaces are uploaded and downloaded", {
   skip_if_no_aws()
-  skip_if_not_installed("arrow")
   s3 <- paws.storage::s3()
   bucket_name <- random_bucket_name()
   s3$create_bucket(Bucket = bucket_name)
@@ -16,7 +15,6 @@ tar_test("aws workspaces are uploaded and downloaded", {
           max_tries = 20
         )
       ),
-      format = "parquet",
       repository = "aws",
       repository_meta = "aws"
     )

Diff for: tests/gcp/test-gcp-workspaces.R

@@ -0,0 +1,43 @@
+# Use sparingly to minimize gcp costs.
+# Verify all `targets` buckets are deleted afterwards.
+tar_test("gcp workspaces are uploaded and downloaded", {
+  skip_if_no_gcp()
+  skip_if_not_installed("arrow")
+  bucket_name <- random_bucket_name()
+  # needs to be a GCP project the tester auth has access to
+  gcp_gcs_auth(max_tries = 5)
+  project <- Sys.getenv("GCE_DEFAULT_PROJECT_ID")
+  googleCloudStorageR::gcs_create_bucket(bucket_name, projectId = project)
+  on.exit(gcp_gcs_delete_bucket(bucket_name))
+  expr <- quote({
+    tar_option_set(
+      resources = tar_resources(
+        gcp = tar_resources_gcp(
+          bucket = !!bucket_name,
+          prefix = "_targets",
+          max_tries = 20
+        )
+      ),
+      repository = "gcp",
+      repository_meta = "gcp"
+    )
+    list(
+      tar_target(x, stop("this is an error"))
+    )
+  })
+  expr <- tar_tidy_eval(expr, environment(), TRUE)
+  eval(as.call(list(`tar_script`, expr, ask = FALSE)))
+  path <- "_targets/workspaces/x"
+  expect_false(gcp_gcs_exists(key = path, bucket = bucket_name))
+  expect_error(tar_make(callr_function = NULL), class = "tar_condition_run")
+  expect_true(gcp_gcs_exists(key = path, bucket = bucket_name))
+  unlink(path)
+  expect_false(file.exists(path))
+  expect_error(tar_workspace(x), class = "tar_condition_validate")
+  tar_workspace_download(x)
+  expect_true(file.exists(path))
+  expect_silent(tar_workspace(x))
+  tar_destroy()
+  expect_false(gcp_gcs_exists(key = path, bucket = bucket_name))
+  expect_error(tar_workspace_download(x), class = "http_404")
+})