Skip to content

Commit 52779f8

Browse files
gilbertocamaragilbertocamara
authored
Merge pull request #1109 from M3nin0/feature/sources
datasources: CDSE as datasource and new DEAfrica products
2 parents f865305 + f20f76f commit 52779f8

26 files changed

+3010
-1447
lines changed

‎DESCRIPTION

+3
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ Imports:
6464
torch (>= 0.11.0),
6565
utils
6666
Suggests:
67+
aws.s3,
6768
caret,
6869
cli,
6970
covr,
@@ -130,6 +131,7 @@ Collate:
130131
'api_data.R'
131132
'api_debug.R'
132133
'api_download.R'
134+
'api_environment.R'
133135
'api_factory.R'
134136
'api_file_info.R'
135137
'api_file.R'
@@ -166,6 +168,7 @@ Collate:
166168
'api_source.R'
167169
'api_source_aws.R'
168170
'api_source_bdc.R'
171+
'api_source_cdse.R'
169172
'api_source_deafrica.R'
170173
'api_source_hls.R'
171174
'api_source_local.R'

‎NAMESPACE

+13
Original file line numberDiff line numberDiff line change
@@ -126,20 +126,26 @@ S3method(.reg_s2tile_convert,grd_cube)
126126
S3method(.reg_s2tile_convert,rtc_cube)
127127
S3method(.slice_dfr,numeric)
128128
S3method(.source_collection_access_test,"mpc_cube_sentinel-1-grd")
129+
S3method(.source_collection_access_test,cdse_cube)
129130
S3method(.source_collection_access_test,mpc_cube)
130131
S3method(.source_collection_access_test,stac_cube)
131132
S3method(.source_collection_access_test,usgs_cube)
132133
S3method(.source_cube,stac_cube)
134+
S3method(.source_filter_tiles,"cdse_cube_sentinel-1-rtc")
133135
S3method(.source_filter_tiles,"mpc_cube_sentinel-1-grd")
134136
S3method(.source_filter_tiles,stac_cube)
135137
S3method(.source_item_get_bands,stac_cube)
138+
S3method(.source_item_get_cloud_cover,cdse_cube)
136139
S3method(.source_item_get_cloud_cover,sdc_cube)
137140
S3method(.source_item_get_cloud_cover,stac_cube)
141+
S3method(.source_item_get_date,cdse_cube)
142+
S3method(.source_item_get_date,deafrica_cube)
138143
S3method(.source_item_get_date,stac_cube)
139144
S3method(.source_item_get_hrefs,bdc_cube)
140145
S3method(.source_item_get_hrefs,sdc_cube)
141146
S3method(.source_item_get_hrefs,stac_cube)
142147
S3method(.source_item_get_hrefs,usgs_cube)
148+
S3method(.source_items_bands_select,cdse_cube)
143149
S3method(.source_items_bands_select,stac_cube)
144150
S3method(.source_items_cube,stac_cube)
145151
S3method(.source_items_fid,stac_cube)
@@ -150,25 +156,31 @@ S3method(.source_items_new,"mpc_cube_sentinel-1-rtc")
150156
S3method(.source_items_new,"mpc_cube_sentinel-2-l2a")
151157
S3method(.source_items_new,aws_cube)
152158
S3method(.source_items_new,bdc_cube)
159+
S3method(.source_items_new,cdse_cube)
153160
S3method(.source_items_new,deafrica_cube)
154161
S3method(.source_items_new,deafrica_cube_s2_l2a)
155162
S3method(.source_items_new,hls_cube)
156163
S3method(.source_items_new,sdc_cube)
157164
S3method(.source_items_new,usgs_cube)
158165
S3method(.source_items_tile,"aws_cube_landsat-c2-l2")
166+
S3method(.source_items_tile,"cdse_cube_sentinel-1-rtc")
159167
S3method(.source_items_tile,"mpc_cube_landsat-c2-l2")
160168
S3method(.source_items_tile,"mpc_cube_sentinel-1-grd")
161169
S3method(.source_items_tile,"mpc_cube_sentinel-1-rtc")
162170
S3method(.source_items_tile,"mpc_cube_sentinel-2-l2a")
163171
S3method(.source_items_tile,aws_cube)
164172
S3method(.source_items_tile,bdc_cube)
173+
S3method(.source_items_tile,cdse_cube)
165174
S3method(.source_items_tile,deafrica_cube)
175+
S3method(.source_items_tile,deafrica_cube_rainfall_chirps_daily)
176+
S3method(.source_items_tile,deafrica_cube_rainfall_chirps_monthly)
166177
S3method(.source_items_tile,hls_cube)
167178
S3method(.source_items_tile,sdc_cube)
168179
S3method(.source_items_tile,usgs_cube)
169180
S3method(.source_roi_tiles,"mpc_cube_landsat-c2-l2")
170181
S3method(.source_roi_tiles,sdc_cube)
171182
S3method(.source_roi_tiles,stac_cube)
183+
S3method(.source_tile_get_bbox,"cdse_cube_sentinel-1-rtc")
172184
S3method(.source_tile_get_bbox,"mpc_cube_sentinel-1-grd")
173185
S3method(.source_tile_get_bbox,"mpc_cube_sentinel-1-rtc")
174186
S3method(.source_tile_get_bbox,stac_cube)
@@ -397,6 +409,7 @@ S3method(summary,sits_accuracy)
397409
S3method(summary,sits_area_accuracy)
398410
export("sits_bands<-")
399411
export("sits_labels<-")
412+
export(.mpc_clean_token_cache)
400413
export(impute_linear)
401414
export(sits_accuracy)
402415
export(sits_accuracy_summary)

‎R/api_conf.R

+51-1
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,6 @@
132132

133133
return(yml_file)
134134
}
135-
136135
#' @title Return the internal configuration file (only for developers)
137136
#' @name .conf_internals_file
138137
#' @keywords internal
@@ -146,6 +145,57 @@
146145
.check_that(file.exists(yml_file))
147146
return(yml_file)
148147
}
148+
#' @title Return the message configuration files (only for developers)
149+
#' @name .conf_sources_files
150+
#' @keywords internal
151+
#' @noRd
152+
#' @return internal sources configuration
153+
.conf_sources_files <- function() {
154+
.check_set_caller(".conf_sources_files")
155+
# list the source files configurations
156+
package_files <- system.file("extdata", "sources", package = "sits")
157+
yml_files <- list.files(
158+
path = package_files,
159+
pattern = "config_source_*",
160+
full.names = TRUE
161+
)
162+
# check that the file name is valid
163+
purrr::map(yml_files, .check_file)
164+
return(yml_files)
165+
}
166+
#' @name .conf_load_sources
167+
#' @description Loads sources configurations
168+
#' @keywords internal
169+
#' @noRd
170+
#' @return NULL, called for side effects
171+
.conf_load_sources <- function() {
172+
# get file paths
173+
source_yml_files <- .conf_sources_files()
174+
# load files
175+
source_configs <- purrr::map(source_yml_files, function(file) {
176+
yaml::yaml.load_file(
177+
input = file,
178+
merge.precedence = "override"
179+
)
180+
})
181+
# prepare sources object
182+
source_obj <- purrr::map(source_configs, "sources")
183+
source_obj <- purrr::flatten(source_obj)
184+
# prepare extras objects (e.g., token, url config)
185+
extras_obj <- purrr::map(source_configs, function(source_config) {
186+
source_config[["sources"]] <- NULL
187+
source_config
188+
})
189+
extras_obj <- purrr::flatten(extras_obj)
190+
# merge objects
191+
config_obj <- utils::modifyList(extras_obj, list(
192+
sources = source_obj
193+
))
194+
# set configurations
195+
do.call(.conf_set_options, args = config_obj)
196+
# done
197+
return(invisible(NULL))
198+
}
149199
#' @title Return the message configuration file (only for developers)
150200
#' @name .conf_messages_file
151201
#' @keywords internal

‎R/api_environment.R

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
2+
# ---- Environment operations ----
3+
#' @title Function to patch environment variables (Developer only).
4+
#' @keywords internal
5+
#' @noRd
6+
#'
7+
#' @description
8+
#' This function patches environment variables, swapping them from a `source`
9+
#' to a `target` variable. You can rollback the swap operation using the
10+
#' `.environment_rollback.`
11+
#'
12+
#' @note
13+
#' This function is suitable only for internal `sits` tests and should not be
14+
#' used for any other purpose.
15+
#' @param env_config List with the configuration of the environment.
16+
#' @return Called for side effects.
17+
.environment_patch <- function(env_config) {
18+
env_prefix <- env_config[["name"]]
19+
env_variables <- env_config[["variables"]]
20+
21+
purrr::map(1:length(env_variables), function(var_idx) {
22+
var_source <- names(env_variables)[[var_idx]]
23+
var_target <- unname(env_variables)[[var_idx]]
24+
# Get current value of the target variable
25+
var_target_value <- Sys.getenv(var_target)
26+
# Save current value of the target variable in the "swap area"
27+
var_target_swap <- paste(env_prefix, var_target, sep = "_SWAP_")
28+
var_target_swap_value <- list(tmp = var_target_value)
29+
var_target_swap_value <- stats::setNames(
30+
var_target_swap_value,
31+
var_target_swap
32+
)
33+
# Save variable
34+
do.call(Sys.setenv, var_target_swap_value)
35+
# Get the new value of the target variable
36+
var_source_value <- Sys.getenv(var_source)
37+
# Save new value in the target variable
38+
var_target_new_value <- list(tmp = var_source_value)
39+
var_target_new_value <- stats::setNames(
40+
var_target_new_value,
41+
var_target
42+
)
43+
# Save variable
44+
do.call(Sys.setenv, var_target_new_value)
45+
})
46+
return(invisible(NULL))
47+
}
48+
49+
#' @title Function to rollback patch in environment variables (Developer only).
50+
#' @keywords internal
51+
#' @noRd
52+
#'
53+
#' @description
54+
#' This function rollback patches in environment variables created with the
55+
#' function `.environment_patch`.
56+
#' @note
57+
#' This function is suitable only for internal `sits` tests and should not be
58+
#' used for any other purpose.
59+
#' @param env_config List with the configuration of the environment.
60+
#' @return Called for side effects.
61+
.environment_rollback <- function(env_config) {
62+
env_prefix <- env_config[["name"]]
63+
env_variables <- env_config[["variables"]]
64+
65+
purrr::map(1:length(env_variables), function(var_idx) {
66+
var_source <- names(env_variables)[[var_idx]]
67+
var_target <- unname(env_variables)[[var_idx]]
68+
# Get current value of the target variable
69+
var_target_value <- Sys.getenv(var_target)
70+
# Save current value in the source variable
71+
var_source_new_value <- list(tmp = var_target_value)
72+
var_source_new_value <- stats::setNames(
73+
var_source_new_value,
74+
var_source
75+
)
76+
do.call(Sys.setenv, var_source_new_value)
77+
# Get current value of the target variable in the "swap area"
78+
var_target_swap <- paste(env_prefix, var_target, sep = "_SWAP_")
79+
var_target_swap_value <- Sys.getenv(var_target_swap)
80+
# Prepare current target
81+
var_target_swap_value <- list(tmp = var_target_swap_value)
82+
var_target_swap_value <- stats::setNames(
83+
var_target_swap_value,
84+
var_target
85+
)
86+
do.call(Sys.setenv, var_target_swap_value)
87+
})
88+
return(invisible(NULL))
89+
}
90+
91+
# ---- Environment configurations ----
92+
#' @title Function to create patch configuration for the CDSE source.
93+
#' @keywords internal
94+
#' @noRd
95+
#'
96+
#' @description
97+
#' This function creates a configuration for the `patch` and `rollback`
98+
#' specialized for the CDSE requirements.
99+
#' @note
100+
#' This function is suitable only for internal `sits` tests and should not be
101+
#' used for any other purpose.
102+
#' @return List with the configuration of the CDSE environment.
103+
.environment_cdse <- function() {
104+
# Define environment name. This name is used as a prefix for the
105+
# "swap area" which is used to keep values saved
106+
env_name <- "CDSE"
107+
# Create the environment variables
108+
env_variables <- list(
109+
"CDSE_ACCESS_KEY_ID" = "AWS_ACCESS_KEY_ID",
110+
"CDSE_SECRET_ACCESS_KEY" = "AWS_SECRET_ACCESS_KEY",
111+
"CDSE_S3_ENDPOINT" = "AWS_S3_ENDPOINT",
112+
"CDSE_VIRTUAL_HOSTING" = "AWS_VIRTUAL_HOSTING"
113+
)
114+
# Create a base environment definition
115+
env_definition <-
116+
list(name = env_name, variables = env_variables)
117+
# Define a class, based on the name
118+
class(env_definition) <- c(env_name, "list")
119+
# Done!
120+
env_definition
121+
}

‎R/api_plot_raster.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@
7878
n = n_colors,
7979
palette = palette,
8080
title = band,
81-
midpoint = NA
81+
midpoint = NA,
82+
style.args = list(na.rm = TRUE)
8283
) +
8384
tmap::tm_graticules(
8485
labels.size = as.numeric(.conf("tmap", "graticules_labels_size"))

‎R/api_regularize.R

+3
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,9 @@
201201
dplyr::group_map(~{
202202
# prepare a sf object representing the bbox of each image in file_info
203203
cube_crs <- dplyr::filter(cube, .data[["crs"]] == .x$crs)
204+
if (nrow(cube_crs) == 0) {
205+
cube_crs <- cube
206+
}
204207
fi_bbox <- .bbox_as_sf(.bbox(
205208
x = .fi(cube_crs),
206209
default_crs = .crs(cube_crs),

0 commit comments

Comments
 (0)