From ae4ac5e08ea4cab20559e2a34e13ebdadcdc7a27 Mon Sep 17 00:00:00 2001 From: Eli Pousson Date: Tue, 24 Sep 2024 20:50:35 -0400 Subject: [PATCH 01/17] Fix #220 by adding informative warning --- R/arc-select.R | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/R/arc-select.R b/R/arc-select.R index 31cf847..b222fc9 100644 --- a/R/arc-select.R +++ b/R/arc-select.R @@ -267,6 +267,17 @@ collect_layer <- function( sf::st_crs(res) <- sf::st_crs(x) } + if (nrow(res) < n_feats) { + # See https://github.com/R-ArcGIS/arcgislayers/issues/110 + cli::cli_warn( + c( + "Results include fewer than the expected {n_feats} features.", + "*" = "Try setting {.arg page_size} to a smaller value to make + sure results include all available features." + ) + ) + } + res } From c0a776d2d93620d8cfa01a1f8791a84f2d5ac5b4 Mon Sep 17 00:00:00 2001 From: Eli Pousson Date: Tue, 24 Sep 2024 20:52:55 -0400 Subject: [PATCH 02/17] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 117e176..dbac6bb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,7 @@ ## Bug fixes - `page_size` resulted in error due to introduction of type-check. Fixed and added test to avoid in the future. [#205](https://github.com/R-ArcGIS/arcgislayers/issues/205) +- Add warning if `arc_select()` results include fewer features than expected from request [#220](https://github.com/R-ArcGIS/arcgislayers/issues/220) ## New features From ea19cea93db240411c05f4afd73994ebdcf64c6c Mon Sep 17 00:00:00 2001 From: Eli Pousson Date: Thu, 26 Sep 2024 00:01:18 -0400 Subject: [PATCH 03/17] Add helper functions to clear default query in URL --- R/arc-open.R | 8 ++++++-- R/utils.R | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/R/arc-open.R b/R/arc-open.R index 8be2d1b..f0d4191 100644 --- a/R/arc-open.R +++ b/R/arc-open.R @@ -57,6 +57,10 @@ arc_open <- function(url, token = arc_token()) { check_url(url) + # parse url query and strip from url if query matches default + query <- parse_url_query(url) + url <- clear_url_query(url) + # extract layer metadata meta <- fetch_layer_metadata(url, token) @@ -84,12 +88,12 @@ arc_open <- function(url, token = arc_token()) { "FeatureLayer" = structure( meta, class = layer_class, - query = list() + query = query ), "Table" = structure( meta, class = layer_class, - query = list() + query = query ), "FeatureServer" = structure( meta, diff --git a/R/utils.R b/R/utils.R index 6d0eb4a..342a88a 100644 --- a/R/utils.R +++ b/R/utils.R @@ -216,3 +216,50 @@ data_frame <- function(x, call = rlang::caller_env()) { check_data_frame(x, call = call) structure(x, class = c("tbl", "data.frame")) } + +#' @noRd +clear_url_query <- function(url, keep_default = FALSE) { + query <- parse_url_query(url, keep_default = keep_default) + + if (rlang::is_empty(query)) { + return(url) + } + + url_elements <- httr2::url_parse(url) + + # Rebuild URL without query + paste0( + url_elements[["scheme"]], "://", + url_elements[["hostname"]], + sub("/query$", "", url_elements[["path"]]) + ) +} + +#' @noRd +parse_url_query <- function(url, keep_default = FALSE) { + # Parse url + url_elements <- httr2::url_parse(url) + + # Return empty list if no query is included in url + if (is.null(url_elements[["query"]])) { + return(list()) + } + + # Check for default query values + query_match <- match( + url_elements[["query"]], + c( + list(outFields = "*", where = "1=1", f = "geojson"), + list(outFields = "*", where = "1=1") + ) + ) + + # Return empty list for default query + if (!is.numeric(query_match) && !keep_default) { + invisible(list()) + } + + # Otherwise return query + url_elements[["query"]] +} + From 12f57d9a7f607996c624f4d2818969a4044ae695 Mon Sep 17 00:00:00 2001 From: Eli Pousson Date: Thu, 26 Sep 2024 00:17:20 -0400 Subject: [PATCH 04/17] Return NULL if query is default Replace NULL value w/ list() in arc_open --- R/arc-open.R | 2 +- R/utils.R | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/R/arc-open.R b/R/arc-open.R index f0d4191..fdacffb 100644 --- a/R/arc-open.R +++ b/R/arc-open.R @@ -58,7 +58,7 @@ arc_open <- function(url, token = arc_token()) { check_url(url) # parse url query and strip from url if query matches default - query <- parse_url_query(url) + query <- parse_url_query(url) %||% list() url <- clear_url_query(url) # extract layer metadata diff --git a/R/utils.R b/R/utils.R index 342a88a..28c22e0 100644 --- a/R/utils.R +++ b/R/utils.R @@ -221,7 +221,7 @@ data_frame <- function(x, call = rlang::caller_env()) { clear_url_query <- function(url, keep_default = FALSE) { query <- parse_url_query(url, keep_default = keep_default) - if (rlang::is_empty(query)) { + if (!is.null(query) && rlang::is_empty(query)) { return(url) } @@ -254,9 +254,9 @@ parse_url_query <- function(url, keep_default = FALSE) { ) ) - # Return empty list for default query - if (!is.numeric(query_match) && !keep_default) { - invisible(list()) + # Return NULL for default query + if (is.numeric(query_match) && !keep_default) { + return(NULL) } # Otherwise return query From 6cc851b3c7415de449f17e7d526ade889065a96e Mon Sep 17 00:00:00 2001 From: Eli Pousson Date: Thu, 26 Sep 2024 00:17:39 -0400 Subject: [PATCH 05/17] Add test for handling URL w/ default query --- tests/testthat/test-arc_open.R | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-arc_open.R b/tests/testthat/test-arc_open.R index 96a51cd..735993f 100644 --- a/tests/testthat/test-arc_open.R +++ b/tests/testthat/test-arc_open.R @@ -4,8 +4,15 @@ test_that("arc_open(): Feature Layer", { ft_url <- "https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/USA_Counties_Generalized_Boundaries/FeatureServer/0" - expect_no_error(arc_open(ft_url)) + lyr <- arc_open(ft_url) + expect_no_error(lyr) + + ft_query_url <- "https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/USA_Counties_Generalized_Boundaries/FeatureServer/0/query?outFields=%2A&where=1%3D1" + + lyr_q <- arc_open(ft_query_url) + + expect_identical(lyr, lyr_q) }) From 0b3d5b68e22a2ef0d6a98d70164706ca900e0488 Mon Sep 17 00:00:00 2001 From: Eli Pousson Date: Thu, 26 Sep 2024 00:19:22 -0400 Subject: [PATCH 06/17] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index dbac6bb..02f17dd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ - `arc_raster()` gains an argument `raster_fn` which takes a character scalar and performs a raster function server side before returning results - `list_service_raster_fns()` is a new helper function to list available raster functions for an `ImageServer` +- `arc_open()` ignores default queries included in input URLs and retains custom queries in place of `query` argument. ([#215](https://github.com/R-ArcGIS/arcgislayers/issues/215)) ## Breaking changes From 9e0121b87ed52cee22445db0408b7ebb22016a86 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 26 Sep 2024 10:17:07 -0700 Subject: [PATCH 07/17] update news --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 02f17dd..31c531a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,7 +9,7 @@ - `arc_raster()` gains an argument `raster_fn` which takes a character scalar and performs a raster function server side before returning results - `list_service_raster_fns()` is a new helper function to list available raster functions for an `ImageServer` -- `arc_open()` ignores default queries included in input URLs and retains custom queries in place of `query` argument. ([#215](https://github.com/R-ArcGIS/arcgislayers/issues/215)) +- `arc_open()` ignores queries included in input URLs and retains any custom queries in the `query` attribute for `Table` and `FeatureLayer`s. ([#215](https://github.com/R-ArcGIS/arcgislayers/issues/215)) ## Breaking changes From d8cb9654db8974f408909e41394fe740594c16c5 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 26 Sep 2024 16:32:01 -0700 Subject: [PATCH 08/17] Increment version number to 0.3.1 --- DESCRIPTION | 6 +++--- NEWS.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 49b1d4c..aaa4b67 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: arcgislayers Type: Package Title: An Interface to ArcGIS Data Services -Version: 0.3.0.9000 +Version: 0.3.1 Authors@R: c( person("Josiah", "Parry", , "josiah.parry@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9910-865X")), @@ -21,9 +21,9 @@ Encoding: UTF-8 LazyData: true Imports: arcgisutils (>= 0.2.0), - arcpbf (>= 0.1.2), + arcpbf (>= 0.1.5), cli, - httr2 (>= 1.0.0), + httr2 (>= 1.0.5), jsonify, lifecycle, RcppSimdJson, diff --git a/NEWS.md b/NEWS.md index 31c531a..9f640ad 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -# arcgislayers (development version) +# arcgislayers 0.3.1 ## Bug fixes From 0abc81c39940f9438b777d98d92fa9bec8378923 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 26 Sep 2024 16:32:11 -0700 Subject: [PATCH 09/17] update cran comments --- cran-comments.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cran-comments.md b/cran-comments.md index 5cb7034..48b0216 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -2,13 +2,7 @@ 0 errors | 0 warnings | 1 note -* This is a new release with small bug fixes and 2 minor new functions. - ------ - -Previous notes: - -* ArcGIS is a brand name and not a software name which is why it is unquoted some places in the DESCRIPTION's. These being the `Title` field and in the `Description` where it says "ArcGIS location services." -* REST is not a software but an architecture so it is not quoted. -* Removed R from the title per request -* Replaced instances of `if (interactive())` with `\dontrun{}`. All of these examples require an authorization token that is generated interactively as well as modifies remote data sources. As such +This is a re-release of the removed `arcgislayers`. +This package depends on `arcpbf` which has a minimum supported rust version +of 1.70. +This package's check **will fail** on CRAN's Fedora machines. \ No newline at end of file From e831c51405395da280bdfe43f83145fd9cd404d4 Mon Sep 17 00:00:00 2001 From: Martha Bass Date: Thu, 3 Oct 2024 09:54:54 -0500 Subject: [PATCH 10/17] Update pkgdown --- _pkgdown.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index f7f634b..900b8fc 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -5,3 +5,43 @@ template: navbar: structure: right: [search, github] + +reference: + +- title: Read Services + contents: + - arc_open + - arc_raster + - arc_read + - arc_select + - get_layer + - get_all_layers + - get_layers + - get_layer_estimates + - query_layer_attachments + - download_attachments +- title: Publish Services + contents: + - create_feature_server + - xss_defaults + - add_item + - publish_item + - publish_layer + - .publish_params +- title: Edit Services + contents: + - add_features + - update_features + - delete_features + - truncate_layer +- title: Utilities + contents: + - clear_query + - list_fields + - pull_field_aliases + - list_items + - refresh_layer + - prepare_spatial_filter + - match_spatial_rel + - update_params + \ No newline at end of file From 84caba93d1848975cc729a8776c35bde299330da Mon Sep 17 00:00:00 2001 From: Martha Bass Date: Thu, 3 Oct 2024 10:08:41 -0500 Subject: [PATCH 11/17] Update doc --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index aaa4b67..a8bd9aa 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -32,7 +32,7 @@ Imports: terra, utils Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 Suggests: dbplyr, dplyr, From 400b7fb0cf743998b01201188e51a262b6350f04 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 3 Oct 2024 08:26:13 -0700 Subject: [PATCH 12/17] update workflow to add quarto as a dep --- .github/workflows/pkgdown.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pkgdown.yml b/.github/workflows/pkgdown.yml index ed7650c..a507a6b 100644 --- a/.github/workflows/pkgdown.yml +++ b/.github/workflows/pkgdown.yml @@ -32,7 +32,7 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@v2 with: - extra-packages: any::pkgdown, local::. + extra-packages: any::pkgdown, any::quarto, local::. needs: website - name: Build site From 283e5c1321abc4980b2c399a2145327f70f524e3 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 3 Oct 2024 11:29:36 -0400 Subject: [PATCH 13/17] Update _pkgdown.yml --- _pkgdown.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index 900b8fc..e73dfa9 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -20,6 +20,7 @@ reference: - get_layer_estimates - query_layer_attachments - download_attachments + - list_service_raster_fns - title: Publish Services contents: - create_feature_server From e453efa4ac2f3df3fe4f0cdc7b8fabf894fb07bd Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 3 Oct 2024 08:34:59 -0700 Subject: [PATCH 14/17] remove old notes in articles --- vignettes/articles/esri-geometry-notes.qmd | 94 ---------------------- 1 file changed, 94 deletions(-) delete mode 100644 vignettes/articles/esri-geometry-notes.qmd diff --git a/vignettes/articles/esri-geometry-notes.qmd b/vignettes/articles/esri-geometry-notes.qmd deleted file mode 100644 index 1d793bc..0000000 --- a/vignettes/articles/esri-geometry-notes.qmd +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: "Esri Geometry Objects" -format: html ---- - -- Feature objects are different than geometry objects. - - geometry objects do not have attributes - - feature objects do have attributes - -https://developers.arcgis.com/documentation/common-data-types/feature-object.htm - -### Features - -- Features consist of 2 properties both of which are optional - - attributes - - field values can be: string number, or boolean - - cannot be arrays - - geometry - - a geometry object - - https://developers.arcgis.com/documentation/common-data-types/geometry-objects.htm -- while both are optional, they both need to be included in the json the following are valid - -``` json - -{ - "geometry": { - }, - "attributes": { - } -} -``` - -``` json -{ - "geometry": { - "x": -81.49822900952586, - "y": 36.43139560823739 - }, - "attributes": { - } -} -``` - -``` json -{ - "geometry": { - "x": -81.49822900952586, - "y": 36.43139560823739 - }, - "attributes": { - "date": "2023-01-04" - } -} -``` - -### Feature Sets - -- `featureSet` is created from feature objects - - tables do not return geometry -- spatial references: - - should be set at the top level - - if not set at the top level then it takes the spatial reference of the first feature's geometry object - - note that geometry objects don't have to have spatial reference - - if neither are set it takes on `UnknownCoordinateSystem` - - by default, sf and sfc object should set a top level spatial reference -- 3 optional fields: - - `objectIdFieldName` - - `globalIdFieldName` - - `displayFieldName` - -### Spatial References - -https://developers.arcgis.com/documentation/common-data-types/geometry-objects.htm - -- can either be a `wkid` or `wkt` - -- `wkid` is the well-known ID e.g. 4326 - -- `wkt` is the well-known text format - -- `st_crs()` will always return the wkt, that should be used. - -- esriGeometry can also support Vertical Coordinate Systems (VCS) these should be supported, but later - -https://developers.arcgis.com/documentation/common-data-types/featureset-object.htm - -- adding a crs uses a lot of text so it should be done only once when possible. so its important to consider when this is done. - - -## Development priority? - -- add features from sf object - - top level CRS - - registered as feature set From 3ea96425e779cc76a6b72c9d4f1699a73c134170 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Thu, 3 Oct 2024 17:14:31 -0700 Subject: [PATCH 15/17] document list service rastyer fns --- .gitignore | 1 + CRAN-SUBMISSION | 3 --- NAMESPACE | 1 + R/raster-fns.R | 12 ++++++++++-- man/list_service_raster_fns.Rd | 9 ++++++++- 5 files changed, 20 insertions(+), 6 deletions(-) delete mode 100644 CRAN-SUBMISSION diff --git a/.gitignore b/.gitignore index c2324a7..30fc236 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +CRAN-SUBMISSION .httr-oauth .Rprofile .Rprofile diff --git a/CRAN-SUBMISSION b/CRAN-SUBMISSION deleted file mode 100644 index a793ec8..0000000 --- a/CRAN-SUBMISSION +++ /dev/null @@ -1,3 +0,0 @@ -Version: 0.3.0 -Date: 2024-07-05 14:30:23 UTC -SHA: f3d74e25fa88bb6a82d53e1390d00ffa543972fd diff --git a/NAMESPACE b/NAMESPACE index e6c6536..dbb8d5e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -31,6 +31,7 @@ export(get_layer_estimates) export(get_layers) export(list_fields) export(list_items) +export(list_service_raster_fns) export(match_spatial_rel) export(prepare_spatial_filter) export(publish_item) diff --git a/R/raster-fns.R b/R/raster-fns.R index 3974c29..592e4ab 100644 --- a/R/raster-fns.R +++ b/R/raster-fns.R @@ -6,9 +6,17 @@ #' @inheritParams arcgisutils::infer_esri_type #' @param x an `ImageServer`. #' @returns a data.frame of the available raster functions. +#' @export #' @examples -#' # example code -#' +#' # use paste to avoid cran note +#' furl <- paste0( +#' "https://di-usfsdata.img.arcgis.com/arcgis/rest/services", +#' "/FIA_BIGMAP_2018_Species_Aboveground_Biomass/ImageServer" +#' ) +#' +#' service <- arc_open(furl) +#' raster_fns <- list_service_raster_fns(service) +#' head(raster_fns) list_service_raster_fns <- function(x, arg = rlang::caller_arg(x), call = rlang::caller_call()) { check_inherits_any(x, "ImageServer") diff --git a/man/list_service_raster_fns.Rd b/man/list_service_raster_fns.Rd index 0f0597c..262edfc 100644 --- a/man/list_service_raster_fns.Rd +++ b/man/list_service_raster_fns.Rd @@ -37,6 +37,13 @@ This function returns the \code{rasterFunctionInfos} field of the \code{ImageSer as a \code{data.frame}. If the field does not exist then an error is emitted. } \examples{ -# example code +# use paste to avoid cran note +furl <- paste0( + "https://di-usfsdata.img.arcgis.com/arcgis/rest/services", + "/FIA_BIGMAP_2018_Species_Aboveground_Biomass/ImageServer" +) +service <- arc_open(furl) +raster_fns <- list_service_raster_fns(service) +head(raster_fns) } From 3c5970cdbcb358b2912d8cdb2b69aeeffab97904 Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Wed, 16 Oct 2024 16:22:33 -0700 Subject: [PATCH 16/17] remove dplyr compatibility --- .Rbuildignore | 1 + R/zzz.R | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 4795a48..46cc59e 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,3 +15,4 @@ ^cran-comments\.md$ ^CRAN-SUBMISSION$ ^README\.md$ +^R/dplyr-features\.R$ diff --git a/R/zzz.R b/R/zzz.R index 705b8d9..379b030 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -25,11 +25,11 @@ s3_register <- function(pkg, generic, class, fun = NULL) { .onLoad <- function(...) { s3_register("sf", "st_crs", "FeatureLayer") s3_register("sf", "st_crs", "ImageServer") - s3_register("dplyr", "collect", "FeatureLayer") - s3_register("dplyr", "collect", "Table") - s3_register("dplyr", "filter", "Table") - s3_register("dplyr", "filter", "FeatureLayer") - s3_register("dplyr", "select", "FeatureLayer") - s3_register("dplyr", "select", "Table") + # s3_register("dplyr", "collect", "FeatureLayer") + # s3_register("dplyr", "collect", "Table") + # s3_register("dplyr", "filter", "Table") + # s3_register("dplyr", "filter", "FeatureLayer") + # s3_register("dplyr", "select", "FeatureLayer") + # s3_register("dplyr", "select", "Table") } From 26bc0e6659732279cda9e2d7cac894317f2aee3b Mon Sep 17 00:00:00 2001 From: Josiah Parry Date: Wed, 16 Oct 2024 16:25:41 -0700 Subject: [PATCH 17/17] update news.md closes #218 closes #224 closes #111 --- DESCRIPTION | 2 +- NEWS.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index a8bd9aa..5e19659 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: arcgislayers Type: Package Title: An Interface to ArcGIS Data Services -Version: 0.3.1 +Version: 0.3.1.9000 Authors@R: c( person("Josiah", "Parry", , "josiah.parry@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9910-865X")), diff --git a/NEWS.md b/NEWS.md index 9f640ad..1727654 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +# arcgislayers (development version) + +## Breaking changes + +- `dplyr` methods for `collect()`, `select()`, and `filter()` have been removed. + # arcgislayers 0.3.1 ## Bug fixes