From 5f0cbb7c20a3c2a306d60736c0e64859c50e1914 Mon Sep 17 00:00:00 2001 From: Salam Elbilig Date: Thu, 16 Apr 2020 22:28:43 -0700 Subject: [PATCH 1/5] bump Clojure CLI version --- languages/clojure.toml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/languages/clojure.toml b/languages/clojure.toml index 6018c7da..3ef9e113 100644 --- a/languages/clojure.toml +++ b/languages/clojure.toml @@ -10,21 +10,22 @@ packages = [ "openjdk-11-jre-headless" ] setup = [ - "wget https://download.clojure.org/install/linux-install-1.10.1.478.sh", - "chmod +x linux-install-1.10.1.478.sh", - "./linux-install-1.10.1.478.sh", - "rm linux-install-1.10.1.478.sh", + "wget https://download.clojure.org/install/linux-install-1.10.1.536.sh", + "chmod +x linux-install-1.10.1.536.sh", + "./linux-install-1.10.1.536.sh", + "rm linux-install-1.10.1.536.sh", + + # Fetch Maven artifacts eagerly in order to cache them in the container image. + # (https://clojure.org/reference/deps_and_cli#_command_line_tools) + """clojure -Sverbose \ + -Sdeps '{:deps {org.clojure/tools.namespace {:mvn/version \"1.0.0\"}} :mvn/local-repo \"/home/runner/.m2/repository\"}' \ + --eval ''""", - # this ensures that we cache the maven deps in the image ($XDG_CONFIG_HOME) - # https://clojure.org/reference/deps_and_cli#_command_line_tools - # pretty ridiculous but what else can you do? - "clojure -Sverbose -Sdeps '{:mvn/local-repo \"/home/runner/.m2/repository\"}' --eval ''", - "mv /root/.m2/repository $XDG_CONFIG_HOME/clojure/repository && rm -rf /root/.m2", "/usr/bin/build-prybar-lang.sh clojure" ] versionCommand = [ "clojure", - "-e", + "--eval", "(clojure-version)" ] From 4f058a907c6c0b7bac3befe127e7cb6cc6612959 Mon Sep 17 00:00:00 2001 From: Salam Elbilig Date: Thu, 16 Apr 2020 23:45:02 -0700 Subject: [PATCH 2/5] Avoid regenerating classpath cache. Note, when running a new Polygott container, in order for `clj`/`clojure` command to find and use the classpath cache and the Maven artifact cache generated during the time the Polygott container image is built: 1) the container needs to be run as user `runner` in group `runner` (i.e., with `--user runner:runner`) 2) `polygott-lang-setup -l clojure` needs to be run first inside the container before running any other Polygott command (e.g., `polygott-self-test`, `polygott-survey`, and `run-project`) --- languages/clojure.toml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/languages/clojure.toml b/languages/clojure.toml index 3ef9e113..30c4f616 100644 --- a/languages/clojure.toml +++ b/languages/clojure.toml @@ -15,23 +15,29 @@ setup = [ "./linux-install-1.10.1.536.sh", "rm linux-install-1.10.1.536.sh", - # Fetch Maven artifacts eagerly in order to cache them in the container image. + # Trigger downloading of Clojure JARs now to avoid downloading them at runtime. # (https://clojure.org/reference/deps_and_cli#_command_line_tools) - """clojure -Sverbose \ - -Sdeps '{:deps {org.clojure/tools.namespace {:mvn/version \"1.0.0\"}} :mvn/local-repo \"/home/runner/.m2/repository\"}' \ - --eval ''""", + """clojure \ + -Sdeps '{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."] :mvn/local-repo "/home/runner/.m2/repository"}' \ + -Sforce \ + -Sverbose \ + --eval ''""", "/usr/bin/build-prybar-lang.sh clojure" ] versionCommand = [ - "clojure", + "clj", + "-Sdeps", + """{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."] :mvn/local-repo "/home/runner/.m2/repository"}""", "--eval", "(clojure-version)" ] [run] command = [ - "clojure", + "clj", + "-Sdeps", + """{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."] :mvn/local-repo "/home/runner/.m2/repository"}""", "main.clj" ] From 364d1cdeee974a540ccb61a22ae9014ee7cefd93 Mon Sep 17 00:00:00 2001 From: Salam Elbilig Date: Fri, 17 Apr 2020 02:12:42 -0700 Subject: [PATCH 3/5] Run `clojure` as `runner` So that 1) the classpath cache is generated with the correct roots relative to `/home/runner` 2) the Maven artifact cache is stored in `/home/runner` 3) we don't have to use the `mvn/local-repo` trick, which we would have to use with Prybar in order for it to find and use the classpath cache and the Maven artifact cache --- languages/clojure.toml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/languages/clojure.toml b/languages/clojure.toml index 30c4f616..1d1ba465 100644 --- a/languages/clojure.toml +++ b/languages/clojure.toml @@ -17,18 +17,19 @@ setup = [ # Trigger downloading of Clojure JARs now to avoid downloading them at runtime. # (https://clojure.org/reference/deps_and_cli#_command_line_tools) - """clojure \ - -Sdeps '{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."] :mvn/local-repo "/home/runner/.m2/repository"}' \ - -Sforce \ - -Sverbose \ - --eval ''""", + """su runner \ + -c 'clojure \ + -Sdeps "{:deps {org.clojure/tools.namespace {:mvn/version \\\"1.0.0\\\"}} :paths [\\\"src\\\" \\\".\\\"]}" \ + -Sforce \ + -Sverbose \ + --eval ""'""", "/usr/bin/build-prybar-lang.sh clojure" ] versionCommand = [ "clj", "-Sdeps", - """{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."] :mvn/local-repo "/home/runner/.m2/repository"}""", + """{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."]}""", "--eval", "(clojure-version)" ] @@ -37,7 +38,7 @@ versionCommand = [ command = [ "clj", "-Sdeps", - """{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."] :mvn/local-repo "/home/runner/.m2/repository"}""", + """{:deps {org.clojure/tools.namespace {:mvn/version "1.0.0"}} :paths ["src" "."]}""", "main.clj" ] From b12dba501f143410bd19b3238916f033157812ae Mon Sep 17 00:00:00 2001 From: Salam Elbilig Date: Fri, 17 Apr 2020 02:28:11 -0700 Subject: [PATCH 4/5] Run container as runner; set up language before test --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7199aa97..5b208915 100644 --- a/Makefile +++ b/Makefile @@ -11,17 +11,17 @@ image-%: ## Build Docker image with single language LANG .PHONY: run run: image ## Build and run image with all languages - docker run -it --rm polygott + docker run -it --rm --user runner:runner polygott run-%: image-% ## Build and run image with single language LANG - docker run -it --rm polygott-$(*) + docker run -it --rm --user runner:runner polygott-$(*) .PHONY: test test: image ## Build and test all languages - docker run polygott:latest bash -c polygott-self-test + docker run --user runner:runner polygott:latest bash -c polygott-self-test test-%: image-% ## Build and test single language LANG - docker run polygott-$(*) bash -c polygott-self-test + docker run --user runner:runner polygott-$(*) bash -c polygott-self-test .PHONY: changed-test changed-test: $(addprefix test-,$(basename $(notdir $(shell git diff --name-only origin/master -- languages)))) ## Build and test only changed/added languages From bd82da1f404b75d39a1cb86f4d8187e9419be4d7 Mon Sep 17 00:00:00 2001 From: Salam Elbilig Date: Fri, 17 Apr 2020 10:49:19 -0700 Subject: [PATCH 5/5] Set up languages first before running their tests. This is to ensure we take advantage of the various caches (e.g., the classpath cache, Maven artifact cache, etc.) that are generated while building the container image. --- gen/inside-test.ejs | 15 ++++++++------- gen/tests.ejs | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/gen/inside-test.ejs b/gen/inside-test.ejs index 4eee436d..9fd2963e 100644 --- a/gen/inside-test.ejs +++ b/gen/inside-test.ejs @@ -2,17 +2,18 @@ CODE=0 -<% for ( let lang of languages ) { -%> -# <%= lang.name %> -<% for ( let tname in lang.tests ) { -%> -<% let test = lang.tests[tname]; -%> -<% if ( test.input ) continue %> +<% for ( let lang of languages ) { %> +# <%= lang.name %> +<% for ( let tname in lang.tests ) { %> +<% let test = lang.tests[tname]; %> +<% if ( test.input ) continue %> <% if ( test.skip ) { %> echo S <%= lang.name %>:<%= tname %> <% } else { %> +polygott-lang-setup -l <%= lang.id %> echo "<%= btoa(test.code) %>" | base64 --decode | run-project -s -l <%= lang.id %> | diff -u --label "<%= lang.name %>" <( echo "<%= btoa(test.output) %>" | base64 --decode ) - && echo ✓ <%= lang.name %>:<%= tname %> || CODE=1 +<% } %> +<% } %> <% } %> -<% } %> -<% } -%> exit $CODE diff --git a/gen/tests.ejs b/gen/tests.ejs index a28a79ac..2f2ad491 100644 --- a/gen/tests.ejs +++ b/gen/tests.ejs @@ -1,12 +1,13 @@ -<% for ( let lang of languages ) { -%> -# <%= lang.name %> -<% for ( let tname in lang.tests ) { -%> -<% let test = lang.tests[tname]; -%> -<% if ( test.input ) continue %> -<% if ( test.skip ) { %> +<% for ( let lang of languages ) { %> +# <%= lang.name %> +<% for ( let tname in lang.tests ) { %> +<% let test = lang.tests[tname]; %> +<% if ( test.input ) continue %> +<% if ( test.skip ) { %> echo S <%= lang.name %>:<%= tname %> -<% } else { %> +<% } else { %> +polygott-lang-setup -l <%= lang.id %> echo "<%= btoa(test.code) %>" | base64 --decode | docker run --rm -i polygott run-project -s -l <%= lang.id %> | diff -u --label "<%= lang.name %>" <( echo "<%= btoa(test.output) %>" | base64 --decode ) - && echo ✓ <%= lang.name %>:<%= tname %> -<% } %> +<% } %> <% } %> -<% } -%> +<% } %>