tar_repository_cas_local_gc()

Author: wlandau

NAMESPACE

@@ -491,6 +491,7 @@ export(tar_read_raw)
 export(tar_renv)
 export(tar_repository_cas)
 export(tar_repository_cas_local)
+export(tar_repository_cas_local_gc)
 export(tar_reprex)
 export(tar_resources)
 export(tar_resources_aws)

NEWS.md

@@ -10,7 +10,7 @@
 * Avoid saving a file in `_targets/objects` for `error = "null"`. Instead, switch to a special `"null"` storage format class if `error` is `"null"` the target throws an error. This should allow users to more freely create new formats with `tar_format()` without worrying about how to handle `NULL` objects created by `error = "null"`.
 * Implement `format = "auto"` (#1311, @hadley).
 * Replace `pingr` dependency with `base::socketConnection()` for local URL utilities (#1317, #1318, @Adafede).
-* Implement `tar_repository_cas()` and `tar_repository_cas_local()` for content-addressable storage (#1232, #1314, @noamross).
+* Implement `tar_repository_cas()`, `tar_repository_cas_local()`, and `tar_repository_cas_local_gc()` for content-addressable storage (#1232, #1314, @noamross).
 
 # targets 1.7.1
 

R/class_options.R

@@ -303,7 +303,10 @@ options_class <- R6::R6Class(
       self$repository %|||% "local"
     },
     get_repository_meta = function() {
-      (self$repository_meta %|||% self$repository) %|||% "local"
+      (self$repository_meta %|||% 
+         if_any(is_repository_cas(self$repository), "local", self$repository)
+      ) %|||%
+        "local"
     },
     get_iteration = function() {
       self$iteration %|||% "vector"
@@ -501,7 +504,11 @@ options_class <- R6::R6Class(
       tar_assert_repository(repository)
     },
     validate_repository_meta = function(repository_meta) {
-      tar_assert_repository(repository_meta)
+      tar_assert_in(
+        repository_meta,
+        choices = c("local", "aws", "gcp"),
+        msg = "repository_meta must be one of \"local\", \"aws\", or \"gcp\"."
+      )
     },
     validate_iteration = function(iteration) {
       tar_assert_flag(iteration, c("vector", "list", "group"))

R/tar_option_set.R

@@ -12,9 +12,13 @@
 #' @inheritSection tar_target Storage formats
 #' @inheritParams tar_target
 #' @param repository_meta Character of length 1 with the same values as
-#'   `repository` (`"aws"`, `"gcp"`, `"local"`). Cloud repository
+#'   `repository` but excluding content-addressable storage
+#'   (`"aws"`, `"gcp"`, `"local"`). Cloud repository
 #'   for the metadata text files in `_targets/meta/`, including target
-#'   metadata and progress data. Defaults to `tar_option_get("repository")`.
+#'   metadata and progress data. Defaults to `tar_option_get("repository")`
+#'   except in the case of content-addressable storage (CAS).
+#'   When `tar_option_get("repository")` is a CAS repository,
+#'   the default value of `repository_meta` is `"local"`.
 #' @param imports Character vector of package names.
 #'   For every package listed, `targets` tracks every
 #'   dataset and every object in the package namespace

R/tar_repository_cas.R

@@ -1,10 +1,11 @@
 #' @title Define a custom content-addressable storage
 #'   (CAS) repository (an experimental feature).
 #' @export
-#' @family storage
+#' @family content-addressable storage
 #' @description Define a custom storage repository that uses
 #'   content-addressable storage (CAS).
-#' @details Without content-addressable storage (CAS),
+#' @section Content-addressable storage:
+#'   Without content-addressable storage (CAS),
 #'   the output data of a pipeline is organized based
 #'   on the names of the targets. For example,
 #'   if your pipeline has a target `x`,
@@ -178,7 +179,7 @@
 #'     tar_target(z, write_file(y), format = "file", repository = repository)
 #'   )
 #' })
-#' tar_make(callr_function = NULL)
+#' tar_make()
 #' tar_read(y)
 #' tar_read(z)
 #' list.files("cas")

R/tar_repository_cas_local.R

@@ -1,10 +1,11 @@
 #' @title Local content-addressable storage (CAS) repository
 #'   (an experimental feature).
 #' @export
-#' @family storage
+#' @family content-addressable storage
 #' @description Local content-addressable storage (CAS) repository.
 #' @details Pass to the `repository` argument of [tar_target()] or
 #'   [tar_option_set()] to use a local CAS system.
+#' @inheritSection tar_repository_cas Content-addressable storage
 #' @return A character string from [tar_repository_cas()] which may be
 #'   passed to the `repository` argument of [tar_target()] or
 #'   [tar_option_set()] to use a local CAS system.
@@ -32,7 +33,7 @@
 #'     tar_target(z, write_file(y), format = "file", repository = repository)
 #'   )
 #' })
-#' tar_make(callr_function = NULL)
+#' tar_make()
 #' tar_read(y)
 #' tar_read(z)
 #' list.files("cas")

R/tar_repository_cas_local_gc.R

@@ -0,0 +1,43 @@
+#' @title Local CAS garbage collection
+#' @export
+#' @family content-addressable storage
+#' @description Garbage collection for a local content-addressable
+#'   storage system.
+#' @details Deletes all the files in the local CAS which are not in
+#'   `tar_meta(targets_only = TRUE)$data`, including all locally saved
+#'   historical data of the pipeline. This clears disk space, but
+#'   at the expense of removing historical data and data from
+#'   other colleagues who worked on the same project.
+#' @inheritSection tar_repository_cas Content-addressable storage
+#' @return `NULL` (invisibly). Called for its side effects.
+#'   Removes files from the CAS repository at `path`.
+#' @inheritParams tar_meta
+#' @inheritParams tar_repository_cas_local
+#' @examples
+#' if (identical(Sys.getenv("TAR_EXAMPLES"), "true")) { # for CRAN
+#' tar_dir({ # tar_dir() runs code from a temp dir for CRAN.
+#' tar_script({
+#'   tar_option_set(seed = NA, repository = tar_repository_cas_local())
+#'   list(tar_target(x, sample.int(n = 9e9, size = 1)))
+#' })
+#' for (index in seq_len(3)) tar_make(reporter = "silent")
+#' list.files("_targets/cas")
+#' tar_repository_cas_local_gc()
+#' list.files("_targets/cas")
+#' tar_meta(names = any_of("x"), fields = any_of("data"))
+#' })
+#' }
+tar_repository_cas_local_gc <- function(
+  path = file.path(targets::tar_config_get("store"), "cas"),
+  store = targets::tar_config_get("store")
+) {
+  meta <- targets::tar_meta(
+    fields = tidyselect::any_of("data"),
+    targets_only = TRUE,
+    store = store
+  )
+  keys <- list.files(path)
+  remove <- setdiff(keys, meta$data)
+  unlink(file.path(path, remove), force = TRUE, recursive = TRUE)
+  invisible()
+}

R/tar_target.R

@@ -144,6 +144,8 @@
 #'     See the cloud storage section of
 #'     <https://books.ropensci.org/targets/data.html>
 #'     for details for instructions.
+#'   * A character string from [tar_repository_cas()] for content-addressable
+#'     storage.
 #'
 #'   Note: if `repository` is not `"local"` and `format` is `"file"`
 #'   then the target should create a single output file.

_pkgdown.yml

@@ -62,10 +62,13 @@ reference:
   - 'tar_load_raw'
   - 'tar_load_everything'
   - 'tar_objects'
-  - 'tar_repository_cas'
-  - 'tar_repository_cas_local'
   - 'tar_read'
   - 'tar_read_raw'
+- title: Content-addressable storage
+  contents:
+  - 'tar_repository_cas'
+  - 'tar_repository_cas_local'
+  - 'tar_repository_cas_local_gc'
 - title: Metadata
   contents:
   - 'tar_crew'