Serve viewer.js from storage.clerk.garden (#415)

Serve viewer.js from clerk CAS on storage.clerk.garden instead of google bucket. 

Fixes #377: adds support for serving the source map for Clerk's cljs bundle
Fixes #387: avoids a http request on boot which would lead to a failure when offline

Author: Sohalt

.github/workflows/main.yml

@@ -35,15 +35,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - name: 🔐 Google Auth
-        uses: google-github-actions/auth@v0
-        with:
-          credentials_json: ${{ secrets.GCLOUD_SERVICE_KEY }}
-
-      - name: 🔧 Setup Google Cloud SDK
-        uses: google-github-actions/[email protected]
-
       - name: Build and upload viewer resources
+        env:
+          GITHUB_TOKEN: ${{ secrets.CAS_AUTH_TOKEN }}
         run: bb build+upload-viewer-resources
 
   clj-kondo:

.gitignore

@@ -20,3 +20,4 @@ clerk.iml
 .nrepl-port
 .clj-kondo/marick
 .clj-kondo/nextjournal/clerk
+.secrets.env

bb.edn

@@ -1,6 +1,9 @@
 {:min-bb-version "0.9.159"
  :paths ["bb"]
- :deps {io.github.nextjournal/dejavu {:git/sha "4980e0cc18c9b09fb220874ace94ba6b57a749ca"}}
+ :deps {io.github.nextjournal/dejavu {:git/sha "4980e0cc18c9b09fb220874ace94ba6b57a749ca"}
+        io.github.nextjournal/cas-client {:git/sha "534d8c6679cc005fd9ad99cfe3d5aeaa6411aee8"}
+        mvxcvi/multihash {:git/url "https://github.com/borkdude/clj-multihash"
+                          :git/sha "7cd4d1fc1301aeea36d5ba80e65ebe2fdbaaa392"}}
  :tasks
  {:requires
   ([tasks :as t]
@@ -27,10 +30,6 @@
   lint {:doc "Lints project using clj-kondo"
         :task (apply shell "clj-kondo --lint src test" *command-line-args*)}
 
-  release:js {:doc "Uploads the js release and updates the sha reference."
-              :task (clojure "-T:build upload-to-cas :resource viewer.js")
-              :depends [build:js]}
-
   build:static-app {:doc "Builds a static app with default notebooks"
                     :task (apply clojure "-X:demo:nextjournal/clerk" *command-line-args*)}
 

build.clj

@@ -3,8 +3,7 @@
    [babashka.process :as process]
    [clojure.string :as str]
    [clojure.tools.build.api :as b]
-   [nextjournal.cas :as cas]
-   [nextjournal.clerk.render.hashing]
+   [nextjournal.clerk.config :as config]
    [shared]))
 
 (def lib 'io.github.nextjournal/clerk)
@@ -19,7 +18,7 @@
 (defn package-clerk-asset-map [{:as opts :keys [target-dir]}]
   (when-not target-dir
     (throw (ex-info "target dir must be set" {:opts opts})))
-  (let [asset-map (slurp (nextjournal.clerk.render.hashing/get-lookup-url))]
+  (let [asset-map @config/!asset-map]
     (spit (str target-dir java.io.File/separator "clerk-asset-map.edn") asset-map)))
 
 (defn jar [_]
@@ -62,21 +61,3 @@
            (throw e)
            (println "This release was already deployed."))))
   opts)
-
-(defn asserting [val msg] (assert (seq val) msg) val)
-
-(defn upload! [opts file] (:url (cas/upload! opts file)))
-
-(defn update-resource! [opts _resource _ file]
-  (upload! opts file))
-
-(defn get-gsutil [] (str/trim (:out (process/sh ["which" "gsutil"]))))
-
-(def resource->path
-  '{viewer.js "/js/viewer.js"})
-
-(defn upload-to-cas [{:keys [resource]}]
-  (if-let [target (resource->path resource)]
-    (update-resource! {:exec-path (str/trim (asserting (get-gsutil) "Can't find gsutil executable."))
-                       :target-path "gs://nextjournal-cas-eu/data/"} target :uploading (str "build/" resource))
-    (throw (ex-info (str "unsupported resource " resource) {:supported-resources (keys resource->path)}))))

deps.edn

@@ -41,6 +41,7 @@
                               binaryage/devtools {:mvn/version "1.0.3"}
                               cider/cider-nrepl {:mvn/version "0.29.0"}
                               com.clojure-goes-fast/clj-async-profiler {:mvn/version "1.0.3"}
+                              io.github.nextjournal/cas-client {:git/sha "534d8c6679cc005fd9ad99cfe3d5aeaa6411aee8"}
                               org.slf4j/slf4j-nop {:mvn/version "1.7.36"}
                               org.babashka/cli {:mvn/version "0.5.40"}}
                  :extra-paths ["dev" "notebooks"]
@@ -53,8 +54,7 @@
            :test {:extra-deps {nubank/matcher-combinators {:mvn/version "3.5.1"}
                                org.clojure/test.check {:mvn/version "1.1.1"}
                                lambdaisland/kaocha {:mvn/version "1.66.1034"}
-                               io.github.cognitect-labs/test-runner
-                               {:git/tag "v0.5.1" :git/sha "dfb30dd"}}
+                               io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" :git/sha "dfb30dd"}}
                   :extra-paths ["test"]
                   :jvm-opts ["-Dclerk.resource_manifest={\"/js/viewer.js\" \"/js/viewer.js\"}"]
                   :exec-fn kaocha.runner/exec-fn
@@ -70,9 +70,9 @@
                                io.github.nextjournal/clerk-slideshow {:git/sha "5fe197ca0c8f28128379849dc677cd718be71524"}}}
 
            :build {:deps {io.github.nextjournal/clerk {:local/root "."}
+                          io.github.nextjournal/cas-client {:git/sha "534d8c6679cc005fd9ad99cfe3d5aeaa6411aee8"}
                           io.github.clojure/tools.build {:git/tag "v0.6.1" :git/sha "515b334"}
                           io.github.slipset/deps-deploy {:git/sha "b4359c5d67ca002d9ed0c4b41b710d7e5a82e3bf"}
-                          io.github.nextjournal/cas {:git/url "[email protected]:nextjournal/cas"
-                                                     :git/sha "5e8079b720e347b9466db9c2282ce79a125a011c"}}
+                          mvxcvi/multihash {:mvn/version "2.0.3"}}
                    :extra-paths ["bb" "src" "resources"] ;; for loading lookup-url in build
                    :ns-default build}}}

src/nextjournal/clerk/config.clj

@@ -16,12 +16,10 @@
     (when-not (str/blank? prop)
       (read-string prop))))
 
-
-
 (def !asset-map
   ;; In mvn releases, the asset map is available in the artifact
   (delay (or (some-> (io/resource "clerk-asset-map.edn") slurp edn/read-string)
-             (try ((requiring-resolve 'nextjournal.clerk.render.hashing/read-dynamic-asset-map!))
+             (try ((requiring-resolve 'nextjournal.clerk.render.hashing/dynamic-asset-map))
                   (catch Exception e
                     (throw (ex-info "Error reading dynamic asset map"
                                     (or (ex-data e)
@@ -39,7 +37,6 @@
 #_(reset! !resource->url default-resource-manifest)
 #_(reset! !resource->url (-> (slurp lookup-url) edn/read-string))
 
-
 (def ^:dynamic *in-clerk* false)
 
 (def ^:dynamic *bounded-count-limit*

src/nextjournal/clerk/render/hashing.clj

@@ -2,21 +2,13 @@
   "Computes a hash based for Clerk's render cljs bundle."
   {:no-doc true}
   (:require [babashka.fs :as fs]
-            [clojure.edn :as edn]
             [clojure.java.io :as io]
             [clojure.string :as str]
             [nextjournal.dejavu :as djv]))
 
 (def output-dirs ["resources/public/ui"
                   "resources/public/build"])
 
-;; Example link in bucket:
-;; "https://storage.googleapis.com/nextjournal-cas-eu/data/8VwKauX6JACEP3K6ahNmP5p1w7rWdhKzeGXCDrHMnJiVrUxHVxcm3Xj84K2r3fcAKWxMQKzqoFe92osgFEHCuKCtZC"
-
-(def gs-bucket "gs://nextjournal-cas-eu")
-(def storage-base-url "https://storage.googleapis.com/nextjournal-cas-eu")
-
-
 (defn file-set [base-dir]
   (reduce into
           []
@@ -26,7 +18,6 @@
                                         "yarn.lock"])
            (djv/cljs-files (mapv #(fs/file base-dir %) ["src" "resources"]))]))
 
-
 #_(file-set (fs/file "."))
 #_(System/setProperty "nextjournal.dejavu.debug" "1")
 
@@ -38,41 +29,22 @@
       (throw (ex-info "Clerk could note compute `font-end-hash` for cljs bundle." {:base-dir base-dir})))
     (str (djv/file-set-hash base-dir (file-set base-dir)))))
 
-(defn bucket-lookup-url [lookup-hash]
-  (str gs-bucket "/lookup/" lookup-hash))
-
-(defn asset-name [hash suffix]
-  (str "/assets/" hash
-       (when suffix
-         (str "-" suffix))))
-
-(defn get-lookup-url []
-  (str storage-base-url "/lookup/" (front-end-hash)))
-
-#_(get-lookup-url)
-
-(defn read-dynamic-asset-map!
-  "Computes a hash for Clerk's cljs bundle and tries to load the asset manifest for it.
-
-  Used only when Clerk is used as a git dep, should never be called from the jar."
-  []
-  (edn/read-string (try
-                     (slurp (get-lookup-url))
-                     (catch java.io.FileNotFoundException e
-                       (throw (ex-info (str "Clerk could not find dynamic asset map at " (get-lookup-url)) {:url (get-lookup-url)} e))))))
+(defn assets-tag []
+  (str "clerk-assets@" (front-end-hash)))
 
-#_(read-dynamic-asset-map!)
+(defn dynamic-asset-map []
+  {"/js/viewer.js" (str "https://storage.clerk.garden/nextjournal/" (assets-tag) "/viewer.js?immutable=true")})
 
 (defn build+upload-viewer-resources []
-  (let [front-end-hash (front-end-hash)
-        manifest (str (fs/create-temp-file))
-        res (djv/gs-copy (str (bucket-lookup-url front-end-hash)) manifest false)]
-    (when (= res ::djv/not-found)
+  (let [tag (assets-tag)]
+    (when-not ((requiring-resolve 'nextjournal.cas-client/tag-exists?) {:namespace "nextjournal" :tag tag})
+      (println (format "Could not find entry at %s. Building..." tag))
       ((requiring-resolve 'babashka.tasks/run) 'build:js)
-      (let [content-hash (djv/sha512 (slurp "build/viewer.js"))
-            viewer-js-http-link (str storage-base-url (asset-name content-hash "viewer.js"))]
-        (spit manifest {"/js/viewer.js" viewer-js-http-link})
-        (println "Manifest:" (slurp manifest))
-        (println "Coping manifest to" (bucket-lookup-url front-end-hash))
-        (djv/gs-copy manifest (bucket-lookup-url front-end-hash))
-        (djv/gs-copy "build/viewer.js" (str gs-bucket (asset-name content-hash "viewer.js")))))))
+      (println "Uploading...")
+      (let [res ((requiring-resolve 'nextjournal.cas-client/cas-put) {:path "build"
+                                                                      :auth-token (System/getenv "GITHUB_TOKEN")
+                                                                      :namespace "nextjournal"
+                                                                      :tag tag})]
+        (doseq [[k v] res]
+          (println (str k ": " v))))
+      (println "Done"))))