Merge pull request #1590 from M3nin0/feat/huggingface-source

add support to hugging face datasets as source in sits

Author: gilbertocamara

DESCRIPTION

@@ -199,6 +199,7 @@ Collate:
     'api_source_cdse.R'
     'api_source_deafrica.R'
     'api_source_deaustralia.R'
+    'api_source_hf.R'
     'api_source_hls.R'
     'api_source_local.R'
     'api_source_mpc.R'

NAMESPACE

@@ -164,12 +164,14 @@ S3method(.source_collection_access_test,"cdse_cube_sentinel-1-grd")
 S3method(.source_collection_access_test,"mpc_cube_sentinel-1-grd")
 S3method(.source_collection_access_test,alphaearth_cube)
 S3method(.source_collection_access_test,cdse_cube)
+S3method(.source_collection_access_test,hf_cube)
 S3method(.source_collection_access_test,hls_cube)
 S3method(.source_collection_access_test,mpc_cube)
 S3method(.source_collection_access_test,ogh_cube)
 S3method(.source_collection_access_test,stac_cube)
 S3method(.source_collection_access_test,usgs_cube)
 S3method(.source_cube,alphaearth_cube)
+S3method(.source_cube,hf_cube)
 S3method(.source_cube,stac_cube)
 S3method(.source_filter_tiles,"bdc_cube_landsat-1y")
 S3method(.source_filter_tiles,"bdc_cube_landsat-2m")
@@ -242,6 +244,7 @@ S3method(.source_items_tile,bdc_cube)
 S3method(.source_items_tile,cdse_cube)
 S3method(.source_items_tile,deafrica_cube)
 S3method(.source_items_tile,deaustralia_cube)
+S3method(.source_items_tile,hf_cube)
 S3method(.source_items_tile,hls_cube)
 S3method(.source_items_tile,mpc_cube_hlsl30)
 S3method(.source_items_tile,mpc_cube_hlss30)

NEWS.md

@@ -62,6 +62,7 @@ remove deprecated functions. The highlights below are grouped by theme.
 
 ### New data sources and collections
 * Add AlphaEarth (AEF) as a data source
+* Add support for data cubes shared on HuggingFace (`source = "HF:<user>"`)
 * Add support for the Amazonia-1 and GLAD image collections in the Brazil Data
   Cube (BDC)
 * Add support for the `SENTINEL-1-GRD` collection in CDSE

R/api_band.R

@@ -162,8 +162,12 @@
     cube_source <- .cube_source(cube)
     collection <- .cube_collection(cube)
     # find which are possible color composites for the cube
-    comp_source <- .conf("composites", "sources", cube_source)
-    composites <- comp_source[["collections"]][[collection]]
+    # (not all sources have color composites)
+    composites <- NULL
+    if (.conf_exists("composites", "sources", cube_source)) {
+        comp_source <- .conf("composites", "sources", cube_source)
+        composites <- comp_source[["collections"]][[collection]]
+    }
     # for each color composite (in order)
     # see if bands are available
     for (i in seq_along(composites)) {

R/api_conf.R

@@ -113,6 +113,38 @@
     }
     return(invisible(sits_env[["config"]]))
 }
+#' @title Add a source to the configuration of the current session
+#' @name .conf_add_source
+#' @keywords internal
+#' @noRd
+#' @author Felipe Carlos, \email{efelipecarlos@@gmail.com}
+#' @description Define a given source configuration on-the-fly and remains
+#' available until the session ends.
+#' @param source       Data source.
+#' @param source_conf  Configuration of the data source.
+#' @return Called for side effects.
+.conf_add_source <- function(source, source_conf) {
+    # define source configuration
+    source_conf <- stats::setNames(list(source_conf), source)
+    # set source configuration
+    .conf_set_options(sources = source_conf)
+    # prepare object to save in the current session tracker
+    sources_session <- .default(sits_env[["sources_session"]], list())
+    # save extra defined sources to keep track of them until the 
+    # session ends
+    sits_env[["sources_session"]] <- utils::modifyList(sources_session, source_conf)
+    # return!
+    invisible(source)
+}
+#' @title Return the sources defined during the current session
+#' @name .conf_sources_session
+#' @keywords internal
+#' @noRd
+#' @author Felipe Carlos, \email{efelipecarlos@@gmail.com}
+#' @return Configuration of the sources defined during the session.
+.conf_sources_session <- function() {
+    sits_env[["sources_session"]]
+}
 #' @title Return the default configuration file
 #' @name .conf_file
 #' @keywords internal

R/api_download.R

@@ -38,10 +38,7 @@
 .download_asset.derived_cube <- function(asset, roi, res, n_tries,
                                          output_dir) {
     # Extract version
-    fields <- .file_sans_ext(.file_base(.tile_path(asset)))
-    fields <- strsplit(fields, split = "_", fixed = TRUE)
-    fields <- fields[[1L]]
-    version <- fields[[length(fields)]]
+    version <- .file_derived_version(.tile_path(asset))
     # Define filename
     output_file <- .file_derived_name(
         tile = asset,
@@ -64,10 +61,7 @@
 .download_asset.class_cube <- function(asset, roi, res, n_tries,
                                        output_dir) {
     # Extract version
-    fields <- .file_sans_ext(.file_base(.tile_path(asset)))
-    fields <- strsplit(fields, split = "_", fixed = TRUE)
-    fields <- fields[[1L]]
-    version <- fields[[length(fields)]]
+    version <- .file_derived_version(.tile_path(asset))
     # Define filename
     output_file <- .file_derived_name(
         tile = asset,

R/api_file.R

@@ -185,6 +185,31 @@
     )
 }
 
+#' @title Get the version of a derived file
+#' @noRd
+#' @author Felipe Carlos, \email{efelipecarlos@@gmail.com}
+#' @param file   File name
+#' @return       Version name
+.file_derived_version <- function(file) {
+    # get parse info of files produced by sits
+    parse_info <- .conf("results_parse_info_def")
+    # split the file name
+    fields <- strsplit(.file_sans_ext(file), split = "_", fixed = TRUE)[[1L]]
+    # verification - file name fields and parse info fields match
+    is_field_match <- length(fields) == length(parse_info)
+    # verification - file band is in the list of sits results bands
+    is_results_band <- fields[[which(parse_info == "band")]] %in% .conf("sits_results_bands")
+    # verify if the file name follows the sits convention
+    is_sits_file <- is_field_match && is_results_band
+    # if the file name does not follow the sits convention, return 
+    # the default version
+    if (!is_sits_file) {
+        return(.conf("results_version_def"))
+    }
+    # otherwise, return the version name
+    fields[[which(parse_info == "version")]]
+}
+
 #' @title Build a file path for a mosaic of derived cubes
 #' @author Rolf Simoes, \email{rolfsimoes@@gmail.com}
 #' @author Felipe Carvalho, \email{felipe.carvalho@@inpe.br}

R/api_parallel.R

@@ -55,6 +55,32 @@
     )
 }
 
+#' @title Export the sources defined in the session to cluster workers
+#' @name .parallel_export_sources
+#' @keywords internal
+#' @noRd
+#' @author Felipe Carlos, \email{efelipecarlos@@gmail.com}
+#' @description Workers build their own configuration when sits is loaded, and
+#' are not aware of sources defined while sits is running (e.g., data cubes
+#' shared on HuggingFace). This function includes those sources in each worker
+#' of an open cluster.
+#' @return No value, called for side effect.
+.parallel_export_sources <- function() {
+    # get the sources defined during the session
+    sources <- .conf_sources_session()
+    # verify if there are workers and sources
+    if (!.parallel_is_open() || .has_not(sources)) {
+        return(invisible(NULL))
+    }
+    # if so, export sources to cluster workers
+    parallel::clusterCall(
+        cl = sits_env[["cluster"]],
+        fun = .conf_set_options,
+        sources = sources
+    )
+    # return!
+    invisible(NULL)
+}
 #' @title Start a new sits cluster for parallel processing
 #' @name .parallel_start
 #' @keywords internal
@@ -78,6 +104,7 @@
         # export variables to cluster workers only; with workers <= 1 and
         # no cluster, processing is sequential in the main process and
         # there is nothing to export
+        .parallel_export_sources()
         if (.parallel_is_open() && .has(export_vars)) {
             parallel::clusterExport(
                 cl = sits_env[["cluster"]],
@@ -127,6 +154,9 @@
         flag = log,
         output_dir = output_dir
     )
+    # workers build their own configuration when sits is loaded: sources
+    # defined during the session must also be included in them
+    .parallel_export_sources()
     # export export_list
     if (.has(export_vars)) {
         parallel::clusterExport(