Untangle parser from analyzer (#745)

Make the parser step independent of the analyzer. This allows us to skip analysis when caching is disabled for the whole namespace.

Author: mk

notebooks/hello.clj

@@ -2,6 +2,7 @@
 
 ;; Clerk enables a _rich_, local-first notebook experience using standard Clojure namespaces.
 (ns hello
+  {:nextjournal.clerk/no-cache true}
   (:require [nextjournal.clerk :as clerk]))
 
 ;; Here's a visualization of unemployment in the US.

notebooks/hello.md

@@ -0,0 +1,22 @@
+# Hello Markdown 👋
+
+Clerk enables a _rich_, local-first notebook experience using standard
+Clojure namespaces and markdown.
+
+```clojure
+(ns hello-markdown
+  {:nextjournal.clerk/no-cache true
+   :nextjournal.clerk/visibility {:code :fold}}
+  (:require [nextjournal.clerk :as clerk]))
+```
+
+Here's a visualization of unemployment in the US.
+
+```clojure
+^{::clerk/viewer clerk/vl}
+{:width 700 :height 400 :data {:url "https://vega.github.io/vega-datasets/data/us-10m.json"
+ :format {:type "topojson" :feature "counties"}}
+ :transform [{:lookup "id" :from {:data {:url "https://vega.github.io/vega-datasets/data/unemployment.tsv"}
+ :key "id" :fields ["rate"]}}]
+ :projection {:type "albersUsa"} :mark "geoshape" :encoding {:color {:field "rate" :type "quantitative"}}}
+```

src/nextjournal/clerk.clj

@@ -57,7 +57,7 @@
                              (when-let [path (paths/path-in-cwd file-or-ns)]
                                {:file-path path})
                              {:nav-path (webserver/->nav-path file-or-ns)}
-                             (parser/parse-file {:doc? true} file))
+                             (parser/parse-file file))
                       (catch java.io.FileNotFoundException e
                         (throw (ex-info (str "`nextjournal.clerk/show!` could not find the file: `" (pr-str file-or-ns) "`")
                                         {:file-or-ns file-or-ns}
@@ -648,7 +648,7 @@
   (show! "notebooks/onwards.md")
   (show! "notebooks/pagination.clj")
   (show! "notebooks/how_clerk_works.clj")
-  (show! "notebooks/hello_clojurescript.cljs")
+  (show! "notebooks/hello.cljs")
   (show! "notebooks/conditional_read.cljc")
   (show! "src/nextjournal/clerk/analyzer.clj")
   (show! "src/nextjournal/clerk.clj")

src/nextjournal/clerk/analyzer.clj

@@ -1,13 +1,11 @@
 (ns nextjournal.clerk.analyzer
   {:nextjournal.clerk/no-cache true}
-  (:refer-clojure :exclude [hash read-string])
+  (:refer-clojure :exclude [hash])
   (:require [babashka.fs :as fs]
-            [edamame.core :as edamame]
             [clojure.core :as core]
             [clojure.java.io :as io]
             [clojure.set :as set]
             [clojure.string :as str]
-            [clojure.tools.reader :as tools.reader]
             [clojure.tools.analyzer :as ana]
             [clojure.tools.analyzer.ast :as ana-ast]
             [clojure.tools.analyzer.jvm :as ana-jvm]
@@ -85,29 +83,6 @@
 
 #_(rewrite-defcached '(nextjournal.clerk/defcached foo :bar))
 
-(defn auto-resolves [ns]
-  (as-> (ns-aliases ns) $
-    (assoc $ :current (ns-name *ns*))
-    (zipmap (keys $)
-            (map ns-name (vals $)))))
-
-#_(auto-resolves (find-ns 'nextjournal.clerk.parser))
-#_(auto-resolves (find-ns 'cards))
-
-(defn read-string [s]
-  (edamame/parse-string s {:all true
-                           :syntax-quote {:resolve-symbol tools.reader/resolve-symbol}
-                           :readers *data-readers*
-                           :read-cond :allow
-                           :regex #(list `re-pattern %)
-                           :features #{:clj}
-                           :end-location false
-                           :row-key :line
-                           :col-key :column
-                           :auto-resolve (auto-resolves (or *ns* (find-ns 'user)))}))
-
-#_(read-string "(ns rule-30 (:require [nextjournal.clerk.viewer :as v]))")
-
 (defn unresolvable-symbol-handler [ns sym ast-node]
   ast-node)
 
@@ -297,20 +272,6 @@
                   (filter (comp #{:code} :type)
                           blocks)))))
 
-(defn get-block-id [!id->count {:as block :keys [var form type doc]}]
-  (let [id->count @!id->count
-        id (if var
-             var
-             (let [hash-fn #(-> % nippy/fast-freeze sha1-base58)]
-               (symbol (str *ns*)
-                       (case type
-                         :code (str "anon-expr-" (hash-fn (cond-> form (instance? clojure.lang.IObj form) (with-meta {}))))
-                         :markdown (str "markdown-" (hash-fn doc))))))]
-    (swap! !id->count update id (fnil inc 0))
-    (if (id->count id)
-      (symbol (str *ns*) (str (name id) "#" (inc (id->count id))))
-      id)))
-
 (defn ^:private internal-proxy-name?
   "Returns true if `sym` represents a var name interned by `clojure.core/proxy`."
   [sym]
@@ -377,41 +338,21 @@
    (analyze-doc {:doc? true} doc))
   ([{:as state :keys [doc?]} doc]
    (binding [*ns* *ns*]
-     (let [!id->count (atom {})]
+     (let [add-block-id (partial parser/add-block-id (atom {}))]
        (cond-> (reduce (fn [{:as state notebook-ns :ns} {:as block :keys [type text loc]}]
                          (if (not= type :code)
-                           (update state :blocks conj (assoc block :id (get-block-id !id->count block)))
-                           (let [form (try (read-string text)
-                                           (catch Exception e
-                                             (throw (ex-info (str "Clerk analysis failed reading block: "
-                                                                  (ex-message e))
-                                                             {:block block
-                                                              :file (:file doc)}
-                                                             e))))
-                                 form+loc (cond-> form
-                                            (instance? clojure.lang.IObj form)
-                                            (vary-meta merge (cond-> loc
-                                                               (:file doc) (assoc :clojure.core/eval-file
-                                                                                  (str (cond-> (:file doc)
-                                                                                         (instance? java.net.URL (:file doc)) extract-file))))))
-                                 {:as analyzed :keys [ns-effect?]} (cond-> (analyze form+loc)
-                                                                     (:file doc) (assoc :file (:file doc)))
-                                 _ (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*`
-                                     (eval form))
-                                 block-id (get-block-id !id->count (merge analyzed block))
-                                 analyzed (assoc analyzed :id block-id)]
-
+                           (update state :blocks conj (add-block-id block))
+                           (let [{:as form-analysis :keys [ns-effect? form]} (cond-> (analyze (:form block))
+                                                                               (:file doc) (assoc :file (:file doc)))
+                                 block+analysis (add-block-id (merge block form-analysis))]
+                             (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*`
+                               (eval form))
                              (-> state
-                                 (store-info analyzed)
-                                 (track-var->block+redefs analyzed)
-                                 (update :blocks conj (-> block
-                                                          (merge (dissoc analyzed :deps :no-cache? :ns-effect?))
-                                                          (cond->
-                                                              (parser/ns? form) (assoc :ns? true)
-                                                              doc? (assoc :text-without-meta (parser/text-with-clerk-metadata-removed text (ns-resolver notebook-ns))))))
-                                 (cond->
-                                     (and doc? (not (contains? state :ns)))
-                                   (merge (parser/->doc-settings form) {:ns *ns*}))))))
+                                 (store-info block+analysis)
+                                 (track-var->block+redefs block+analysis)
+                                 (update :blocks conj (cond-> (dissoc block+analysis :deps :no-cache? :ns-effect?)
+                                                        (parser/ns? form) (assoc :ns? true)
+                                                        doc? (assoc :text-without-meta (parser/text-with-clerk-metadata-removed text (ns-resolver notebook-ns)))))))))
 
                        (-> state
                            (cond-> doc? (merge doc))
@@ -421,11 +362,9 @@
                        (:blocks doc))
 
          true (dissoc :doc?)
-         doc? (-> parser/add-block-settings
-                  parser/add-open-graph-metadata
-                  parser/filter-code-blocks-without-form))))))
+         doc? parser/filter-code-blocks-without-form)))))
 
-#_(let [parsed (nextjournal.clerk.parser/parse-clojure-string "clojure.core/dec")]
+#_(let [parsed (parser/parse-clojure-string "clojure.core/dec")]
     (build-graph (analyze-doc parsed)))
 
 (defonce !file->analysis-cache
@@ -603,7 +542,7 @@
             analyze-doc-deps
             set-no-cache-on-redefs
             make-deps-inherit-no-cache
-            (dissoc :analyzed-file-set :counter))))))
+            (dissoc :analyzed-file-set :counter)))))) 
 
 (comment
   (reset! !file->analysis-cache {})

src/nextjournal/clerk/config.clj

@@ -8,8 +8,8 @@
       ".clerk/cache"))
 
 (def cache-disabled?
-  (when-let [prop (System/getProperty "clerk.disable_cache")]
-    (not= "false" prop)))
+  (boolean (when-let [prop (System/getProperty "clerk.disable_cache")]
+             (not= "false" prop))))
 
 (def resource-manifest-from-props
   (when-let [prop (System/getProperty "clerk.resource_manifest")]

src/nextjournal/clerk/eval.clj

@@ -135,8 +135,7 @@
           var-from-def? (and var (var? result) (= var (symbol result)))
           no-cache? (or ns-effect?
                         no-cache?
-                        (boolean (seq @!interned-vars))
-                        config/cache-disabled?)]
+                        (boolean (seq @!interned-vars)))]
       (when (and (not no-cache?)
                  (not ns-effect?)
                  freezable?
@@ -167,7 +166,9 @@
 (defn read+eval-cached [{:as doc :keys [blob->result ->analysis-info ->hash]} codeblock]
   (let [{:keys [id form _vars var]} codeblock
         _ (assert id (format "Missing id on codeblock: '%s'." (pr-str codeblock)))
-        {:as form-info :keys [ns-effect? no-cache? freezable?]} (->analysis-info id)
+        {:as form-info :keys [ns-effect? no-cache? freezable?]} (if (:no-cache doc)
+                                                                  (assoc codeblock :no-cache? true)
+                                                                  (->analysis-info id))
         no-cache?      (or ns-effect? no-cache?)
         hash           (when-not no-cache? (or (get ->hash id)
                                                (analyzer/hash-codeblock ->hash doc codeblock)))
@@ -262,30 +263,41 @@
 
 (defn +eval-results
   "Evaluates the given `parsed-doc` using the `in-memory-cache` and augments it with the results."
-  [in-memory-cache {:as parsed-doc :keys [set-status-fn]}]
+  [in-memory-cache {:as parsed-doc :keys [set-status-fn no-cache]}]
   (if (cljs? parsed-doc)
     (process-cljs parsed-doc)
-    (do
-      (when set-status-fn
-        (set-status-fn {:progress 0.10 :status "Analyzing…"}))
-      (let [{:as analyzed-doc :keys [ns]} (analyzer/build-graph
-                                           (assoc parsed-doc :blob->result in-memory-cache))]
-        (binding [*ns* ns]
-          (-> analyzed-doc
-              analyzer/hash
-              eval-analyzed-doc))))))
+    (let [{:as analyzed-doc :keys [ns]}
+
+          (if no-cache
+            parsed-doc
+            (do
+              (when set-status-fn
+                (set-status-fn {:progress 0.10 :status "Analyzing…"}))
+              (-> parsed-doc
+                  (assoc :blob->result in-memory-cache)        
+                  analyzer/build-graph
+                  analyzer/hash)))]
+      (when (and (not-empty (:var->block-id analyzed-doc))
+                 (not ns))
+        (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns]))))
+      (binding [*ns* ns]
+        (eval-analyzed-doc analyzed-doc)))))
 
 (defn eval-doc
   "Evaluates the given `doc`."
   ([doc] (eval-doc {} doc))
-  ([in-memory-cache doc] (+eval-results in-memory-cache doc)))
+  ([in-memory-cache doc]
+   (+eval-results in-memory-cache
+                  (cond-> doc
+                    config/cache-disabled?
+                    (assoc :no-cache true)))))
 
 (defn eval-file
   "Reads given `file` (using `slurp`) and evaluates it."
   ([file] (eval-file {} file))
   ([in-memory-cache file]
    (->> file
-        (parser/parse-file {:doc? true})
+        parser/parse-file
         (eval-doc in-memory-cache))))
 
 #_(eval-file "notebooks/hello.clj")
@@ -296,6 +308,9 @@
   "Evaluated the given `code-string` using the optional `in-memory-cache` map."
   ([code-string] (eval-string {} code-string))
   ([in-memory-cache code-string]
-   (eval-doc in-memory-cache (parser/parse-clojure-string {:doc? true} code-string))))
+   (eval-doc in-memory-cache (parser/parse-clojure-string code-string))))
 
 #_(eval-string "(+ 39 3)")
+
+#_(nextjournal.clerk/show! "notebooks/hello.md")
+

src/nextjournal/clerk/home.clj

@@ -1,5 +1,6 @@
 (ns nextjournal.clerk.home
-  {:nextjournal.clerk/visibility {:code :hide :result :hide}}
+  {:nextjournal.clerk/visibility {:code :hide :result :hide}
+   :nextjournal.clerk/no-cache true}
   (:require [babashka.fs :as fs]
             [clojure.string :as str]
             [nextjournal.clerk :as clerk]