fix: several improvements and fixes

- check and warn if paged remote API endpoints have a next page
- request maximum number of elements per page on remote endpoints to
  to mitigate risk of missing commits, files
- read deployed hash when reading collection config as this info is
  needed almost all the time
- expose all public data on /git/status
- graceful handling of remote connection error in /git/status
- test and fix gitlab:
    - drastically reduce requests to API on incremental updates
- refactor incremental update functions to expect only one parameter

Author: line-o

.existdb.json

@@ -1,7 +1,7 @@
 {
     "servers": {
         "localhost": {
-            "server": "https://localhost:8443/exist",
+            "server": "https://127.0.0.1:8443/exist",
             "user": "admin",
             "password": "",
             "root": "/db/apps/tuttle"

src/data/tuttle.xml

@@ -4,8 +4,8 @@
         <default>true</default>
         <type>github</type>
         <baseurl>https://api.github.com/</baseurl>
-        <repo>tuttle-sample-data</repo>
         <owner>eeditiones</owner>
+        <repo>tuttle-sample-data</repo>
         <token>XXX</token>
         <ref>next</ref>
         <hookuser>admin</hookuser>
@@ -15,9 +15,11 @@
    <collection name="tuttle-sample-gitlab">
         <type>gitlab</type>
         <baseurl>https://gitlab.com/api/v4/</baseurl>
-        <project-id>tuttle-sample-data</project-id>
+        <owner>line-o</owner>
+        <repo>tuttle-sample-data</repo>
+        <project-id>50872175</project-id>
         <token>XXX</token>
-        <ref>master</ref>
+        <ref>main</ref>
         <hookuser>admin</hookuser>
         <hookpasswd></hookpasswd>
     </collection>

src/modules/api.xql

@@ -42,34 +42,31 @@ declare function api:get-status($request as map(*)) {
 };
 
 declare function api:repo-xml ($info as map(*)) as element(repo) {
-    <repo type="{$info?type}" url="{$info?url}"
-        ref="{$info?ref}" collection="{$info?collection}"
-        message="{$info?message}" status="{$info?status}" />
+    element repo {
+        map:for-each($info, function ($name as xs:string, $value) as attribute() {
+            attribute { $attribute } { $value }
+        })
+    }
 };
 
 declare function api:collection-info ($collection as xs:string) as map(*) {
     let $collection-config := api:get-collection-config($collection)
-    let $actions := vcs:get-actions($collection-config?vcs)
-    let $deployed-commit-hash := doc($collection-config?path || "/gitsha.xml")/hash/value/string()
-    let $info := map {
-        'deployed': $deployed-commit-hash,
-        'type': $collection-config?vcs,
-        'ref': $collection-config?ref,
-        'collection': $collection-config?collection
-    }
+    (: hide passwords and tokens :)
+    let $masked := map:remove($collection-config, ("hookpasswd", "token"))
 
     return
         try {
+            let $actions := vcs:get-actions($collection-config?type)
             let $url := $actions?get-url($collection-config)
             let $last-remote-commit := $actions?get-last-commit($collection-config)
             let $remote-short-sha := app:shorten-sha($last-remote-commit?sha)
 
             let $status :=
-                if ($last-remote-commit?sha = "")
+                if ($remote-short-sha = "")
                 then "error"
-                else if (empty($deployed-commit-hash))
+                else if (empty($collection-config?deployed))
                 then "new"
-                else if ($deployed-commit-hash = $remote-short-sha)
+                else if ($collection-config?deployed = $remote-short-sha)
                 then "uptodate"
                 else "behind"
 
@@ -78,15 +75,15 @@ declare function api:collection-info ($collection as xs:string) as map(*) {
                 then "no commit on remote"
                 else "remote found"
 
-            return map:merge(( $info, map {
+            return map:merge(( $masked, map {
                 'url': $url,
                 'remote': $remote-short-sha,
                 'message': $message,
                 'status': $status
             }))
         }
         catch * {
-            map:merge(( $info, map {
+            map:merge(( $masked, map {
                 'message': $err:description,
                 'status': 'error'
             }))
@@ -99,14 +96,14 @@ declare function api:collection-info ($collection as xs:string) as map(*) {
 declare function api:get-hash($request as map(*)) as map(*) {
     try {
         let $collection-config := api:get-collection-config($request?parameters?collection)
-        let $actions := vcs:get-actions($collection-config?vcs)
+        let $actions := vcs:get-actions($collection-config?type)
         let $collection-staging := $collection-config?path || config:suffix() || "/gitsha.xml"
 
         let $last-remote-commit := $actions?get-last-commit($collection-config)
 
         return map {
             "remote-hash": $last-remote-commit?sha,
-            "local-hash": app:production-sha($collection-config?collection),
+            "local-hash": $collection-config?deployed,
             "local-staging-hash": doc($collection-staging)/hash/value/text()
         }
     }
@@ -172,7 +169,7 @@ declare function api:git-pull($request as map(*)) {
                 map { "message" : doc($lockfile)/task/value/text() || " in progress" }
             )
             else (
-                let $actions := vcs:get-actions($config?vcs)
+                let $actions := vcs:get-actions($config?type)
                 let $write-lock := app:lock-write($collection-destination, "git-pull")
                 let $commit :=
                     if ($request?parameters?hash) then (
@@ -206,7 +203,7 @@ declare function api:git-pull-default($request as map(*)) {
                 map { "message" : doc($lockfile)/task/value/text() || " in progress" }
             )
             else (
-                let $actions := vcs:get-actions($config?vcs)
+                let $actions := vcs:get-actions($config?type)
                 let $write-lock := app:lock-write($collection-destination, "git-pull")
                 let $commit :=
                     if ($request?parameters?hash)
@@ -261,8 +258,8 @@ declare function api:git-deploy($request as map(*)) {
                         let $set-permissions := app:set-permission($config?collection)
                         return
                             map {
-                                "sha" : app:production-sha($config?collection),
-                                "message" : "success"
+                                "sha": config:deployed-sha($config?path),
+                                "message": "success"
                             }
                     )
                     else (
@@ -285,8 +282,8 @@ declare function api:git-deploy($request as map(*)) {
                         let $install := repo:install-and-deploy-from-db(concat($collection-staging-uri, "/pkg.xar"))
                         return 
                             map {
-                                "sha" : app:production-sha($config?collection),
-                                "message" : "success"
+                                "sha": config:deployed-sha($config?path),
+                                "message": "success"
                             }
                     )
                 let $remove-staging := xmldb:remove($collection-staging-uri)
@@ -309,9 +306,11 @@ declare function api:git-deploy($request as map(*)) {
 declare function api:get-commits($request as map(*)) as map(*) {
     try {
         let $config := api:get-collection-config($request?parameters?collection)
-        let $actions := vcs:get-actions($config?vcs)
+        let $actions := vcs:get-actions($config?type)
 
-        return $actions?get-commits($config, $request?parameters?count)
+        return map {
+            'commits': $actions?get-commits($config, $request?parameters?count)
+        }
     }
     catch * {
         map {
@@ -329,7 +328,7 @@ declare function api:get-commits($request as map(*)) as map(*) {
 declare function api:get-commits-default($request as map(*)) as map(*) {
     try {
         let $config := api:get-default-collection-config()
-        let $actions := vcs:get-actions($config?vcs)
+        let $actions := vcs:get-actions($config?type)
 
         return map {
             'commits': $actions?get-commits($config, $request?parameters?count)
@@ -355,7 +354,7 @@ declare function api:incremental($request as map(*)) as map(*) {
         let $collection-path := $config?path
         let $lockfile := $collection-path || "/" || config:lock()
         let $collection-destination-sha := $collection-path || "/gitsha.xml"
-        let $actions := vcs:get-actions($config?vcs)
+        let $actions := vcs:get-actions($config?type)
 
         return
         if (not(xmldb:collection-available($collection-path))) then (
@@ -366,21 +365,21 @@ declare function api:incremental($request as map(*)) as map(*) {
         )
         else if (exists($drymode) and $drymode) then
             map {
-                "changes" : $actions?incremental-dry($config, $config?collection),
+                "changes" : $actions?incremental-dry($config),
                 "message" : "success"
             }
         else if (doc-available($lockfile)) then (
             map { "message" : doc($lockfile)/task/value/text() || " in progress" }
         )
         else (
             let $write-lock := app:lock-write($collection-path, "incremental")
-            let $incremental := $actions?incremental($config, $config?collection)
+            let $incremental := $actions?incremental($config)
             let $remove-lock := app:lock-remove($collection-path)
 
             return
                 map {
-                    "sha" : app:production-sha($config?collection),
-                    "message" : "success"
+                    "sha": config:deployed-sha($config?path),
+                    "message": "success"
                 }
         )
     }
@@ -434,14 +433,14 @@ declare function api:hook($request as map(*)) {
                 let $login := xmldb:login($config?path, $config?hookuser, $config?hookpasswd)
                 let $write-lock := app:lock-write($config?path, "hook")
 
-                let $incremental := $actions?incremental($config, $config?collection)
+                let $incremental := $actions?incremental($config)
 
                 let $remove-lock := app:lock-remove($collection-path)
 
-                return 
+                return
                     map {
-                        "sha" : app:production-sha($config?collection),
-                        "message" : "success"
+                        "sha": config:deployed-sha($config?path),
+                        "message": "success"
                     }
             )
     }

src/modules/app.xql

@@ -240,21 +240,6 @@ declare function app:set-permission($collection as xs:string, $path as xs:string
         )
 };
 
-(:~
-: Get Sha of production collection
-:)
-declare function app:production-sha($collection as xs:string) {
-    let $gitsha := config:prefix() || "/" || $collection || "/gitsha.xml"
-
-    return 
-        if (doc($gitsha)/hash/value/text()) then 
-            doc($gitsha)/hash/value/text()
-        else
-            map { 
-                "message" : concat($gitsha, " not exist")
-            }
-};
-
 (:~
  : Helper function of unzip:mkcol() 
 :)
@@ -307,23 +292,24 @@ declare function app:shorten-sha($git-sha as xs:string?) as xs:string? {
 
 declare function app:request-json($request as element(http:request)) {
     let $raw := app:request($request)
-    let $decoded := util:base64-decode($raw)
+    let $decoded := util:base64-decode($raw[2])
     let $json := parse-json($decoded)
 
-    return $json
+    return ($raw[1], $json)
 };
 
 (:~
  : Github request
  :)
 declare function app:request($request as element(http:request)) {
+    (: let $_ := util:log("info", $request/@href) :)
     let $response := http:send-request($request)
     let $status-code := xs:integer($response[1]/@status)
 
     return
         if ($status-code >= 400)
         then error(xs:QName("app:connection-error"), "server connection failed: " || $response[1]/@message || " (" || $status-code || ")", $response[1])
-        else $response[2]
+        else $response
 };
 
 declare function app:extract-archive($zip, $collection as xs:string) {

src/modules/config.xql

@@ -12,6 +12,7 @@ declare variable $config:tuttle-config as element(tuttle) := doc("/db/apps/tuttl
  :)
 declare function config:collections($collection as xs:string) as map(*)? {
     let $collection-config := $config:tuttle-config/repos/collection[@name = $collection]
+    let $path := config:prefix() || $collection
 
     return
         if (empty($collection-config))
@@ -22,18 +23,30 @@ declare function config:collections($collection as xs:string) as map(*)? {
             "repo" : $collection-config/repo/string(),
             "owner" : $collection-config/owner/string(),
             "project-id" : $collection-config/project-id/string(),
-            "vcs": $collection-config/type/string(),
+            "type": $collection-config/type/string(),
             "baseurl": $collection-config/baseurl/string(),
             "ref": $collection-config/ref/string(),
             "collection": $collection-config/@name/string(),
-            "path": config:prefix() || $collection,
             "hookuser":  $collection-config/hookuser/string(),
+
+            "path": $path,
+            "deployed": config:deployed-sha($path),
+
             (: be careful never to expose these :)
             "hookpasswd": $collection-config/hookpasswd/string(),
             "token": config:token($collection-config)
         }
 };
 
+(:~
+: Which commit is deployed?
+:)
+declare function config:deployed-sha($path as xs:string) as xs:string? {
+    if (doc-available($path || "/gitsha.xml"))
+    then doc($path || "/gitsha.xml")/hash/value/string()
+    else ()
+};
+
 declare %private function config:token($collection-config as element(collection)) as xs:string? {
     let $env-var := "tuttle_token_" || replace($collection-config/@name/string(), "-", "_")
     let $token-env := environment-variable($env-var)
@@ -54,57 +67,57 @@ declare function config:collection-config-available($collection as xs:string) as
 (:~
  : List collection names
  :)
-declare function config:list-collections() {
+declare function config:list-collections() as xs:string* {
     $config:tuttle-config/repos/collection/@name/string()
 };
 
 
 (:~
  : Defile default collection
  :)
-declare function config:default-collection(){
+declare function config:default-collection() as xs:string? {
     $config:tuttle-config/repos/collection[default="true"]/@name/string()
 };
 
 (:~
  : Blacklist - these files are not checkout from git and are ignored
  :)
-declare function config:blacklist(){
+declare function config:blacklist() as xs:string* {
     $config:tuttle-config/blacklist/file/string()
 };
 
 (:~
  : Suffix of the checked out git statging collection 
  :)
-declare function config:suffix(){
+declare function config:suffix() as xs:string {
     $config:tuttle-config/config/@suffix/string()
 };
 
 (:~
  : The running task is stored in the lockfile. It ensures that two tasks do not run at the same time. 
  :)
-declare function config:lock(){
+declare function config:lock() as xs:string {
     $config:tuttle-config/config/@lock/string()
 };
 
 (:~
  : Prefix for collections
  :)
-declare function config:prefix(){
+declare function config:prefix() as xs:string {
     $config:tuttle-config/config/@prefix/string()
 };
 
 (:~
  : The destination where the key for the webhook is stored.
  :)
-declare function config:apikeys(){
+declare function config:apikeys() as xs:string* {
     $config:tuttle-config/config/@apikeys/string()
 };
 
 (:~
  : DB User and Permissions as fallback if "permissions" not set in repo.xml
  :)
-declare function config:sm(){
+declare function config:sm() as map(*) {
     let $sm := $config:tuttle-config/config/sm
 
     return map {