tech.ml.dataset example for graalvm-clojure (#39)

* Adding vanilly deps-based tmd pathway.

* Removing large file

Author: cnuernber

Diff for: tech.ml.dataset/.gitignore

@@ -0,0 +1,8 @@
+.cpcache
+*.crc
+*.parquet
+ds-graal
+graalvm*
+pom.xml
+target
+hello-tmd

Diff for: tech.ml.dataset/README.md

@@ -0,0 +1,20 @@
+# Graal Native And Data Frames
+
+This repository is a simple example of how to build a graal native application that
+processes a CSV and both reads and writes to parquet.  With graal native you
+lose some level of performance but you get a single packaged executable
+with very few dependencies and of course instant startup time.
+
+
+## Building
+
+```console
+scripts/compile
+```
+
+
+## Running
+
+```console
+./ds-graal
+```

Diff for: tech.ml.dataset/deps.edn

@@ -0,0 +1,28 @@
+{:paths ["src" "classes" "resources"]
+ :deps {org.clojure/clojure            {:mvn/version "1.10.2"}
+        techascent/tech.ml.dataset     {:mvn/version "5.19"}
+
+        org.apache.parquet/parquet-hadoop {:mvn/version "1.12.0"
+                                           :exclusions [org.slf4j/slf4j-log4j12]}
+        org.apache.hadoop/hadoop-common {:mvn/version "3.3.0"
+                                         :exclusions [org.slf4j/slf4j-log4j12]}
+        ;; We literally need this for 1 POJO formatting object.
+        org.apache.hadoop/hadoop-mapreduce-client-core {:mvn/version "3.3.0"
+                                                        :exclusions [org.slf4j/slf4j-log4j12]}}
+ :aliases
+ {:depstar
+  {:replace-deps {com.github.seancorfield/depstar {:mvn/version "2.0.193"}}
+   :ns-default hf.depstar
+   :exec-fn hf.depstar/uberjar
+   :exec-args {:group-id "graalvm-clojure"
+               :artifact-id "hello-tmd"
+               :version "1.00-beta-1"
+               :sync-pom true
+               :aot true
+               :compile-ns [hello-tmd.main]
+               :main-class hello-tmd.main
+               :jar "target/hello-tmd.jar"
+               ;;Disable tensor code generation and ensure direct linking.
+               :jvm-opts ["-Dtech.v3.datatype.graal-native=true"
+                          "-Dclojure.compiler.direct-linking=true"
+                          "-Dclojure.spec.skip-macros=true"]}}}}

Diff for: tech.ml.dataset/resources/logback.xml

@@ -0,0 +1,13 @@
+<configuration debug="false">
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <!-- encoders are assigned the type
+         ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+    <encoder>
+      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+    </encoder>
+  </appender>
+
+  <root level="info">
+    <appender-ref ref="STDOUT" />
+  </root>
+</configuration>

Diff for: tech.ml.dataset/scripts/activate-graal

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ ! -e graalvm ]; then
+    scripts/get-graal
+fi
+
+export GRAALVM_HOME="$(pwd)/graalvm"
+export PATH="$(pwd)/graalvm/bin:$PATH"
+export JAVA_HOME="$(pwd)/graalvm"

Diff for: tech.ml.dataset/scripts/compile

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+source scripts/activate-graal
+
+rm -rf classes && mkdir classes
+echo "Building uberjar"
+clojure -X:depstar
+
+
+graalvm/bin/native-image \
+    --report-unsupported-elements-at-runtime \
+    --initialize-at-build-time \
+    --no-fallback \
+    --no-server \
+    -H:+ReportExceptionStackTraces \
+    --enable-https \
+    --allow-incomplete-classpath \
+    --initialize-at-run-time=org.apache.hadoop.util.DataChecksum$Java9Crc32CFactory \
+    --initialize-at-run-time=org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder \
+    --initialize-at-run-time=org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder \
+    --initialize-at-run-time=org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder \
+    --initialize-at-run-time=org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder \
+    -J-Dclojure.spec.skip-macros=true \
+    -J-Dclojure.compiler.direct-linking=true \
+    -J-Dtech.v3.datatype.graal-native=true \
+    -jar target/hello-tmd.jar hello-tmd

Diff for: tech.ml.dataset/scripts/get-graal

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+GRAAL_VERSION="21.0.0.2"
+JDK_VERSION="11"
+
+wget https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-$GRAAL_VERSION/graalvm-ce-java$JDK_VERSION-linux-amd64-$GRAAL_VERSION.tar.gz
+tar -xvzf graalvm-ce-java$JDK_VERSION-linux-amd64-$GRAAL_VERSION.tar.gz
+ln -s "$(pwd)/graalvm-ce-java$JDK_VERSION-$GRAAL_VERSION" "$(pwd)/graalvm"
+rm graalvm-ce-java$JDK_VERSION-linux-amd64-$GRAAL_VERSION.tar.gz
+graalvm/bin/gu install native-image
+
+## Dev builds
+
+# wget https://github.com/graalvm/graalvm-ce-dev-builds/releases/download/21.1.0-dev-20210325_0249/graalvm-ce-java16-linux-amd64-dev.tar.gz
+# tar -xvzf graalvm-ce-java16-linux-amd64-dev.tar.gz
+# ln -s "$(pwd)/graalvm-ce-java16-21.1.0-dev" "$(pwd)/graalvm"
+# rm -f graalvm-ce-java16-linux-amd64-dev.tar.gz
+# graalvm/bin/gu install native-image

Diff for: tech.ml.dataset/src/hello_tmd/main.clj

@@ -0,0 +1,17 @@
+(ns hello-tmd.main
+  (:require [tech.v3.dataset :as ds]
+            [tech.v3.datatype.functional :as dfn]
+            [tech.v3.libs.parquet :as parquet])
+  (:gen-class))
+
+
+(defn -main
+  [& args]
+  (let [test-ds (ds/->dataset "https://raw.githubusercontent.com/techascent/tech.ml.dataset/master/test/data/stocks.csv" {:key-fn keyword})]
+    (println test-ds)
+    (println "price mean:" (dfn/mean (test-ds :price)))
+    (parquet/ds->parquet test-ds "stocks.parquet")
+    (println "succesfully wrote stocks.parquet")
+    (let [pds (first (parquet/parquet->ds-seq "stocks.parquet"))]
+      (println "price mean:" (dfn/mean (pds "price"))))
+    0))