Add more user properties

.github/workflows/R-CMD-check.yaml

@@ -8,6 +8,8 @@ on:
 
 name: R-CMD-check
 
+permissions: read-all
+
 jobs:
   R-CMD-check:
     runs-on: ${{ matrix.config.os }}

.github/workflows/lint.yaml

@@ -8,6 +8,8 @@ on:
 
 name: lint
 
+permissions: read-all
+
 jobs:
   lint:
     runs-on: ubuntu-latest

.github/workflows/pkgdown.yaml

@@ -11,6 +11,8 @@ on:
 
 name: pkgdown
 
+permissions: read-all
+
 jobs:
   pkgdown:
     runs-on: ubuntu-latest

.github/workflows/test-coverage.yaml

@@ -8,6 +8,8 @@ on:
 
 name: test-coverage
 
+permissions: read-all
+
 jobs:
   test-coverage:
     runs-on: ubuntu-latest
@@ -23,7 +25,7 @@ jobs:
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          extra-packages: any::covr
+          extra-packages: any::covr, any::xml2
           needs: coverage
 
       - name: Test coverage
@@ -32,13 +34,22 @@ jobs:
           MYSQL_USER: ${{ secrets.MYSQL_USER }}
           MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
         run: |
-          covr::codecov(
+          cov <- covr::package_coverage(
             quiet = FALSE,
             clean = FALSE,
             install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package")
           )
+          covr::to_cobertura(cov)
         shell: Rscript {0}
 
+      - uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: ${{ github.event_name != 'pull_request' && true || false }}
+          file: ./cobertura.xml
+          plugin: noop
+          disable_search: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
       - name: Show testthat output
         if: always()
         run: |

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: tarflow.iquizoo
 Title: Setup "targets" Workflows for "iquizoo" Data Processing
-Version: 3.11.5
+Version: 3.11.5.9000
 Authors@R: c(
     person("Liang", "Zhang", , "[email protected]", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0001-9041-1150")),
@@ -22,10 +22,10 @@ Imports:
     cli,
     data.iquizoo (>= 2024.3.31.2),
     DBI,
+    glue,
     jsonlite,
     memoise,
     rlang (>= 1.0.0),
-    stringr,
     tarchetypes,
     targets
 Suggests:

NAMESPACE

@@ -4,6 +4,7 @@ export(check_source)
 export(fetch_data)
 export(fetch_iquizoo)
 export(fetch_iquizoo_mem)
+export(get_users_props_names)
 export(parse_data)
 export(setup_option_file)
 export(setup_source)

NEWS.md

@@ -1,3 +1,15 @@
+# tarflow.iquizoo (development version)
+
+## New Features
+
+* Added `cache` argument to `tar_prep_iquizoo()` to optionally specify how to cache the fetched configuration data.
+* Support to customize the columns to be kept in the output of `users`. See [get_users_props_names()] for all available columns, and customize it by setting `subset_users_props` argument in `tar_prep_iquizoo()`.
+* Update contents sql template to support setting course name and game name ([#90](https://github.com/psychelzh/tarflow.iquizoo/issues/90)).
+
+## Bug Fixes
+
+* Let `users` update based on progress hashes. This is important when more users are added in projects.
+
 # tarflow.iquizoo 3.11.5
 
 ## Enhancements

R/database.R

@@ -119,7 +119,7 @@ fetch_data <- function(project_id, game_id, ...,
   )
   query <- query %||% read_file(package_file("sql", paste0(what, ".sql")))
   fetch_iquizoo(
-    stringr::str_glue(
+    glue::glue(
       query,
       .envir = env(table_name = table_name)
     ),

R/setup.R

@@ -116,7 +116,7 @@ setup_option_file <- function(path = NULL, overwrite = FALSE, quietly = FALSE) {
     }
     return(invisible())
   }
-  writeLines(stringr::str_glue(my_cnf_tmpl), path)
+  writeLines(glue::glue(my_cnf_tmpl), path)
 }
 
 # helper functions

R/targets.R

@@ -8,10 +8,11 @@
 #' @param params,contents Used as the configuration of data fetching. These two
 #'   arguments are mutually exclusive. If `params` is specified, it will be used
 #'   as parameters to be bound to the query, see [DBI::dbBind()] for more
-#'   details. The default template requires specifying `organization_name` and
-#'   `project_name`, in that order. If `contents` is specified, it should be a
-#'   [data.frame] and will be used directly as the configuration of data
-#'   fetching. Note `contents` should at least contain `project_id` and
+#'   details. The default template requires specifying `organization_name`,
+#'   `project_name`, `course_name` and `game_name`, in that order. Set the
+#'   column as `NA` to skip that parameter. If `contents` is specified, it
+#'   should be a [data.frame] and will be used directly as the configuration of
+#'   data fetching. Note `contents` should at least contain `project_id` and
 #'   `game_id` names.
 #' @param ... For future usage. Should be empty.
 #' @param what What to fetch. There are basically two types of data, i.e., raw
@@ -31,22 +32,28 @@
 #' @param combine Specify which targets to be combined. Note you should only
 #'   specify names from `c("scores", "raw_data", "raw_data_parsed", "indices")`.
 #'   If `NULL`, none will be combined.
+#' @param subset_users_props The subset of user properties to be fetched. See
+#'   [get_users_props_names()] for all the available properties. If `NULL`, all
+#'   properties will be fetched.
 #' @param templates The SQL template files used to fetch data. See
 #'   [setup_templates()] for details.
 #' @param check_progress Whether to check the progress hash. Set it as `FALSE`
 #'   if the project is finalized.
+#' @param cache The cache to be used in [fetch_iquizoo_mem()].
 #' @return A list of target objects.
 #' @export
 tar_prep_iquizoo <- function(params, contents, ...,
                              what = c("raw_data", "scores"),
                              action_raw_data = c("all", "parse", "none"),
                              combine = NULL,
+                             subset_users_props = NULL,
                              templates = setup_templates(),
-                             check_progress = TRUE) {
+                             check_progress = TRUE,
+                             cache = NULL) {
   check_dots_empty()
   check_templates(templates)
   contents <- switch(check_exclusive(params, contents),
-    params = fetch_iquizoo_mem()(
+    params = fetch_iquizoo_mem(cache)(
       read_file(templates$contents),
       params = unname(
         if (!is_empty(params)) as.list(params)
@@ -80,7 +87,7 @@ tar_prep_iquizoo <- function(params, contents, ...,
       expr(unserialize(!!serialize(contents, NULL)))
     ),
     if (check_progress) tar_prep_hash(contents, templates),
-    tar_fetch_users(contents, templates),
+    tar_fetch_users(contents, subset_users_props, templates, check_progress),
     sapply(
       what,
       tar_fetch_data,
@@ -149,21 +156,47 @@ tar_prep_hash <- function(contents, templates = setup_templates()) {
 #'
 #' @param contents The contents structure used as the configuration of data
 #'   fetching.
+#' @param subset_users_props The subset of user properties to be fetched. See
+#'   [get_users_props_names()] for all the available properties. If `NULL`, all
+#'   properties will be fetched.
 #' @param templates The SQL template files used to fetch data. See
 #'   [setup_templates()] for details.
+#' @param check_progress Whether to check the progress hash. Set it as `FALSE`
+#'   if the project is finalized.
 #' @return A list of target objects.
 #' @export
-tar_fetch_users <- function(contents, templates = setup_templates()) {
+tar_fetch_users <- function(contents, subset_users_props = NULL,
+                            templates = setup_templates(),
+                            check_progress = TRUE) {
   check_templates(templates)
+  if (!is.null(subset_users_props)) {
+    users_props <- users_props[users_props$alias %in% subset_users_props, ]
+  }
+  columns <- paste0(glue::glue_data(
+    users_props,
+    ", {table}.{column} AS {alias}"
+  ), collapse = "")
+  project_ids <- as.character(unique(contents$project_id))
   targets::tar_target_raw(
     "users",
-    bquote(
-      fetch_iquizoo(
-        .(read_file(templates[["users"]])),
-        params = list(.(unique(contents$project_id)))
-      ) |>
-        unique()
-    ),
+    as.call(c(
+      quote(`{`),
+      if (check_progress) {
+        bquote(
+          list(..(syms(paste0("progress_hash_", project_ids)))),
+          splice = TRUE
+        )
+      },
+      bquote(
+        fetch_iquizoo(
+          .(glue::glue(read_file(templates[["users"]]),
+            .envir = list(columns = columns)
+          )),
+          params = list(.(unique(contents$project_id)))
+        ) |>
+          unique()
+      )
+    )),
     packages = "tarflow.iquizoo"
   )
 }

R/utils.R

@@ -1,3 +1,13 @@
+# nocov start
+
+#' Get the names of the user properties.
+#'
+#' @return A character vector of the names.
+#' @export
+get_users_props_names <- function() {
+  users_props$alias # nolint
+}
+
 package_file <- function(type, file) {
   system.file(
     type, file,
@@ -8,3 +18,5 @@ package_file <- function(type, file) {
 read_file <- function(file) {
   paste0(readLines(file), collapse = "\n")
 }
+
+# nocov end

_pkgdown.yml

@@ -26,3 +26,4 @@ reference:
   - setup_source
   - setup_option_file
   - check_source
+  - get_users_props_names

data-raw/users_props.R

@@ -0,0 +1,7 @@
+## code to prepare `users_props` dataset goes here
+
+users_props <- readr::read_csv(
+  "data-raw/users_props.csv",
+  show_col_types = FALSE
+)
+usethis::use_data(users_props, overwrite = TRUE, internal = TRUE)

data-raw/users_props.csv

@@ -0,0 +1,13 @@
+table,column,alias
+project_course_config,Name,project_name
+organization_user,RealName,user_name
+organization_user,Gender,user_sex
+organization_user,Birthday,user_dob
+organization_user,IdCard,user_id_card
+organization_user,StudentNumber,user_id_student
+organization_user,Mobile,user_phone
+base_organization,Name,organization_name
+base_organization,Country,organization_country
+base_organization,Province,organization_province
+base_organization,City,organization_city
+base_organization,District,organization_district

inst/pipelines/use_targets.R

@@ -27,10 +27,10 @@ tar_source()
 # source("other_functions.R") # Source other scripts as needed.
 
 params <- tibble::tribble(
-  ~organization_name, ~project_name,
+  ~organization_name, ~project_name, ~course_name, ~game_name,
   # TODO: replace these with your own
-  "Organization1", "Project1",
-  "Organization2", NA # set as NA if you want to fetch all projects
+  "Organization1", "Project1", NA, NA,
+  "Organization2", NA, NA, NA # set as NA to skip that column
 )
 
 # Replace the target list below with your own:

inst/sql/contents.sql

@@ -7,6 +7,7 @@ SELECT
     c.Name course_name,
     c.Period course_period,
     c2.Id game_id,
+    c2.Name game_name,
     c2.ContentType game_type
 FROM
     iquizoo_business_db.project_course_config pcc
@@ -15,4 +16,4 @@ FROM
     INNER JOIN iquizoo_content_db.course_child_config ccc ON ccc.ChildCourseId = cc.Id AND ccc.Deleted <> 1
     INNER JOIN iquizoo_content_db.content c2 ON c2.Id = ccc.ContentId AND c2.ContentType <> 4 AND c2.Deleted <> 1
     INNER JOIN iquizoo_user_db.base_organization bo ON bo.Id = pcc.OrganizationId AND bo.Deleted <> 1
-WHERE bo.Name = ? AND pcc.Name = IFNULL(?, pcc.Name);
+WHERE bo.Name = IFNULL(?, bo.Name) AND pcc.Name = IFNULL(?, pcc.Name) AND c.Name = IFNULL(?, c.Name) AND c2.Name = IFNULL(?, c2.Name);

inst/sql/users.sql

@@ -1,14 +1,16 @@
+-- start from 2024-07-20, columns related to organization structure will not be supported
 SELECT DISTINCT
-    vo.OrganizationUserId user_id,
-    vo.RealName user_name,
-    vo.Gender user_sex,
-    vo.Birthday user_dob,
-    vo.OrganizationName organization_name,
-    vo.GradeName grade_name,
-    vo.ClassName class_name
+    project_course_user.OrganizationUserId user_id
+    { columns }
 FROM
-    iquizoo_business_db.project_course_config pcc
-    INNER JOIN iquizoo_business_db.project_course_user pcu ON pcu.ProjectCourseConfigId = pcc.Id AND pcu.Deleted <>1 AND pcc.Deleted <> 1
-    INNER JOIN iquizoo_business_db.v_organizationuser vo ON vo.OrganizationUserId = pcu.OrganizationUserId
+    iquizoo_business_db.project_course_config
+    INNER JOIN iquizoo_business_db.project_course_user
+    ON project_course_user.ProjectCourseConfigId = project_course_config.Id
+        AND project_course_user.Deleted <> 1 AND project_course_config.Deleted <> 1
+    INNER JOIN iquizoo_user_db.organization_user
+    ON organization_user.Id = project_course_user.OrganizationUserId
+        AND organization_user.Deleted <> 1
+    INNER JOIN iquizoo_user_db.base_organization
+    ON base_organization.Id = organization_user.OrganizationId
 WHERE
-    pcc.Id = ?;
+    project_course_config.Id = ?;