improve netrc handling for HLS collections

Author: M3nin0

R/api_check.R

@@ -2398,3 +2398,51 @@
     .check_set_caller(".check_linkage_method")
     .check_that(linkage %in% .conf("dendro_linkage"))
 }
+#' @title Check netrc file
+#' @description
+#' Check if netrc file exists and if its content is correct
+#' @param attributes    Attributes required from the netrc file
+#' @return Called for side effects
+#' @keywords internal
+#' @noRd
+.check_netrc_gdal <- function(attributes) {
+    .check_set_caller(".check_netrc_gdal")
+    # define if the current GDAL version is reading netrc from env variable
+    is_gdal_reading_netrc <- .gdal_version() >= "3.7.0"
+    # define from where `netrc` file must be loaded
+    # case 1 - gdal environment variable (requires GDAL >= 3.7.0)
+    netrc_from_var <- ifelse(
+        is_gdal_reading_netrc,
+        Sys.getenv("GDAL_HTTP_NETRC_FILE", unset = NA),
+        NA
+    )
+    # case 2 - netrc file stored in user home directory
+    netrc_from_home <- ifelse(
+        .Platform[["OS.type"]] == "windows",
+        .conf("gdal_netrc_file_path_win"),
+        .conf("gdal_netrc_file_path")
+    )
+    # define which netrc file will be used
+    netrc_file <- ifelse(
+        is.na(netrc_from_var),
+        netrc_from_home,
+        netrc_from_var
+    )
+    # if the env variable is used, then, warning users not to set the GDAL
+    # variable using `Sys.setenv`
+    if (!is.na(netrc_from_var)) {
+        warning(.conf("messages", ".check_netrc_gdal_var"))
+    }
+    # check if file exist
+    .check_that(file.exists(netrc_file))
+    # load netrc content
+    netrc_content <- readLines(netrc_file)
+    # check netrc file content
+    .check_that(
+        any(
+            purrr::map_lgl(netrc_content, function(x) {
+                stringr::str_detect(x, attributes)
+            })
+        )
+    )
+}

R/api_gdal.R

@@ -346,3 +346,10 @@
     )
     return(invisible(out_file))
 }
+#' @title Get GDAL Version
+#' @noRd
+#' @returns  GDAL Version
+.gdal_version <- function() {
+    sf_versions <- sf::sf_extSoftVersion()
+    sf_versions[["GDAL"]]
+}

R/api_source_hls.R

@@ -17,12 +17,8 @@
                                        tiles = NULL) {
     # set caller to show in errors
     .check_set_caller(".source_items_new_hls_cube")
-    # NASA EarthData requires a login/password combination
-    if (.Platform[["OS.type"]] == "windows")
-        netrc_path <- .conf("HLS_NETRC_FILE_PATH_WIN")
-    else
-        netrc_path <- .conf("HLS_NETRC_FILE_PATH")
-    .check_that(file.exists(netrc_path))
+    # check netrc file
+    suppressWarnings(.check_netrc_gdal(attributes = .conf("HLS_ACCESS_URL")))
     # convert tiles to a valid STAC query
     if (!is.null(tiles)) {
         roi <- .s2_mgrs_to_roi(tiles)
@@ -81,18 +77,8 @@
 #' @return Called for side effects
 .source_configure_access.hls_cube <- function(source, collection = NULL) {
     .check_set_caller(".source_configure_access_hls_cube")
-    # NASA EarthData requires a login/password combination
-    if (.Platform[["OS.type"]] == "windows")
-        netrc_file <- .conf("HLS_NETRC_FILE_PATH_WIN")
-    else
-        netrc_file <- .conf("HLS_NETRC_FILE_PATH")
-    # does the file exist
-    if (file.exists(netrc_file)) {
-        conf_hls <- utils::read.delim(netrc_file)
-        if (!(.conf("HLS_ACCESS_URL") %in% names(conf_hls)))
-            stop(.conf("messages", ".source_configure_access_hls_cube"))
-    } else {
-        stop(.conf("messages", ".source_configure_access_hls_cube"))
-    }
+    # check netrc file
+    .check_netrc_gdal(attributes = .conf("HLS_ACCESS_URL"))
+    # done!
     return(invisible(source))
 }

inst/extdata/config_internals.yml

@@ -194,6 +194,11 @@ gdal_creation_options: [ "COMPRESS=LZW", "PREDICTOR=2",
 gdalcubes_options: ["COMPRESS=LZW", "PREDICTOR=2",
                      "BIGTIFF=YES", "BLOCKXSIZE=512",
                      "BLOCKYSIZE=512"]
+
+# GDAL netrc file
+gdal_netrc_file_path: "~/.netrc"
+gdal_netrc_file_path_win: "%HOME%\\_netrc"
+
 # gdalcubes parameters
 gdalcubes_chunk_size: [1, 2048, 2048]
 gdalcubes_type_format: "int16"

inst/extdata/config_messages.yml

@@ -32,6 +32,8 @@
 .check_file_missing: "file does not exist:"
 .check_file_writable: "cannot write file to disk"
 .check_find_class: "cube is not properly configured"
+.check_netrc_gdal: "invalid configuration file\nTo learn more, see instructions in Chapter 4 of the online book"
+.check_netrc_gdal_var: "netrc environment variable configuration detected (GDAL_HTTP_NETRC_FILE)\n please, make sure it is available to all R sessions. To learn more, see the instructions in Chapter 4 of the online book"
 .check_gpu_memory: "when using GPU: gpu_memory must be informed"
 .check_gpu_memory_size: "not enough GPU memory - should have at least 2GB memory free to use"
 .check_is_class_cube: "data should be a classified cube - check 'cube' parameter"
@@ -254,7 +256,6 @@
 .source_collection_tile_check: "invalid tiles for collection"
 .source_configure_access: "unable to access collection - service is unavailable"
 .source_configure_access_aws_cube: "a valid AWS_SECRET_ACCESS_KEY is required to access this collection\n if you have one, include it as an enviromental variable"
-.source_configure_access_hls_cube: "missing HLS access configuration\n see instructions in Chapter 4 of on-line book"
 .source_configure_access_usgs_cube: "a valid AWS_SECRET_ACCESS_KEY is required to access USGS collection\n if you have one, include it as an enviromental variable"
 .source_collection_access_test: "enable to access data from the collection"
 .source_collection_access_var_set: "invalid access vars for collection"
@@ -266,7 +267,7 @@
 .source_items_cube_stac_cube: "error when opening files"
 .source_items_fid_stac_cube: "invalid feature id value"
 .source_items_new: "search returned no items - check selection parameters."
-.source_items_new_hls_cube: "could not find .netrc file\n please configure your access to NASA EarthData"
+.source_items_new_hls_cube: "search returned no items - check selection parameters."
 .source_items_new_cdse_cube: "search returned no items - check selection parameters."
 .source_items_new_mpc_cube_landsat_c2_l2: "when retrieving Landsat collection in MPC searching by tiles is not allowed, use roi"
 .source_items_new_mpc_s1_grd: "invalid orbit parameter for MPC S1 GRD collection"

inst/extdata/sources/config_source_hls.yml

@@ -2,8 +2,7 @@
 # The parameters enable access to the cloud collections
 
 HLS_ACCESS_URL: "machine.urs.earthdata.nasa.gov"
-HLS_NETRC_FILE_PATH: "~/.netrc"
-HLS_NETRC_FILE_PATH_WIN: "%HOME%\\_netrc"
+
 sources:
     HLS                     :
         s3_class            : ["hls_cube", "stac_cube", "eo_cube",