Merge pull request #408 from INCATools/odk.1.2.27

All changes for ODK 1.2.27 at once
INCATools · Apr 4, 2021 · 278e812 · 278e812
2 parents b8028cd + e02c571
commit 278e812
Show file tree

Hide file tree

Showing 18 changed files with 930 additions and 208 deletions.
diff --git a/Changes.md b/Changes.md
@@ -1,3 +1,15 @@
+# v1.2.27 (4 April 2021)
+- revived odklite image with a minimum install (smaller in size than odkfull, with just robot and owltools!)
+- revived robot image with just robot installed
+- Added Jena 3.12.0 to odkfull image
+- Added module configs (see docs)
+- added a complete new documentation system for ontologies using mkdocs (see docs)
+- added OWL2 DL profile checking (see [here](tests/test-robot-validate-profile.yaml) for example)
+- Added sqlite3, dos2unix, and aha to odkfull image
+- Added a `use_custom_import_module` feature (see docs for example)
+- Added ability to add completely customised release artefacts (see Uberon repo for example)
+- Revised update mechanism. Please run the `make update_repo` *multiple times if you encounter problems and to ensure that all changes are picked up*.
+
 # v1.2.26 (10 February 2021): HOTFIXES
 - Hotfixes:
   - The new mireot module technique was buggy and is therefore removed again. Sorry; we will try again next time. You can still use the `custom` option to implement mireot yourself!

diff --git a/Dockerfile b/Dockerfile
@@ -39,6 +39,7 @@ RUN apt-get update &&\
     aha \
     dos2unix \
     sqlite3 \
+    libjson-perl \
     xlsx2csv &&\
     cd /usr/local/bin \
     && ln -s /usr/bin/python3 python \
@@ -94,6 +95,11 @@ ENV FASTOBO_VALIDATOR v0.4.0
 RUN wget https://dl.bintray.com/fastobo/fastobo-validator/$FASTOBO_VALIDATOR/fastobo_validator-x86_64-linux-musl.tar.gz -O- | tar xzC /tools \
 && chmod +x /tools/fastobo-validator
 
+###### JENA ######
+ENV JENA 3.12.0
+RUN wget http://archive.apache.org/dist/jena/binaries/apache-jena-$JENA.tar.gz -O- | tar xzC /tools 
+ENV PATH "/tools/apache-jena-$JENA/bin:$PATH"
+
 ##### Ammonite #####
 # LAYERSIZE ~31MB
 RUN (echo "#!/usr/bin/env sh" \
@@ -139,6 +145,12 @@ RUN cd /tools/ && chmod +x /tools/obodash && git clone --depth 1 --branch docker
 #    ls -l /tools/
 #RUN chmod +x /tools/droid
 
+# ######
+# RUN cd /tools/ && git clone --depth 1 https://github.com/cmungall/blipkit.git && cd blipkit &&\
+# ./configure && make && make install && cd /tools/ && rm -rf blipkit
+# ENV PATH "/usr/local/bin:$PATH"
+
+
 ### 5. Install ODK
 ARG ODK_VERSION=0.0.0
 ENV ODK_VERSION=${ODK_VERSION}
@@ -153,5 +165,6 @@ COPY template/ /tools/templates/
 RUN chmod +x /tools/*.py
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
+RUN echo "PLEASE DONT FORGET TO UPDATE odklite (docker/odklite/Dockerfile) and robot (docker/robot/Dockerfile) images!"
 
 CMD python /tools/odk.py
diff --git a/Makefile b/Makefile
@@ -38,58 +38,83 @@ schema/project-schema.json:
 	./odk/odk.py dump-schema > $@
 
 # Building docker image
-VERSION = "v1.2.26"
+VERSION = "v1.2.27"
 IM=obolibrary/odkfull
+IMLITE=obolibrary/odklite
 DEV=obolibrary/odkdev
-ROBOT_JAR="https://build.obolibrary.io/job/ontodev/job/robot/job/cmd-metrics/6/artifact/bin/robot.jar"
-ROBOT_JAR_ARGS=
-#--build-arg ROBOT_JAR=$(ROBOT_JAR)
+ROB=obolibrary/robot
+#ROBOT_JAR="https://github.com/monarch-ebi-dev/odk_utils/raw/master/robot_maven_test.jar"
+ROBOT_JAR_ARGS=#--build-arg ROBOT_JAR=$(ROBOT_JAR)
 
 docker-build-no-cache:
 	@docker build  --build-arg ODK_VERSION=$(VERSION) $(ROBOT_JAR_ARGS) --no-cache -t $(IM):$(VERSION) . \
-	&& docker tag $(IM):$(VERSION) $(IM):latest
+	&& docker tag $(IM):$(VERSION) $(IM):latest && docker tag $(IM):$(VERSION) $(DEV):latest && \
+	docker build -f docker/odklite/Dockerfile -t $(IMLITE):$(VERSION) . \
+	&& docker tag $(IMLITE):$(VERSION) $(IMLITE):latest && cd docker/robot/ && make docker-build
 
 docker-build:
 	@docker build --build-arg ODK_VERSION=$(VERSION)  $(ROBOT_JAR_ARGS)  -t $(IM):$(VERSION) . \
-	&& docker tag $(IM):$(VERSION) $(IM):latest
-
-docker-build-use-cache-dev:
+	&& docker tag $(IM):$(VERSION) $(IM):latest && docker tag $(IM):$(VERSION) $(DEV):latest && \
+	docker build -f docker/odklite/Dockerfile -t $(IMLITE):$(VERSION) . \
+	&& docker tag $(IMLITE):$(VERSION) $(IMLITE):latest && cd docker/robot/ && make docker-build
+
+docker-build-dev:
 	@docker build --build-arg ODK_VERSION=$(VERSION) -t $(DEV):$(VERSION) . \
 	&& docker tag $(DEV):$(VERSION) $(DEV):latest
 
-docker-run:
-	docker run --rm -ti --name odkfull $(IM)
-
 docker-clean:
 	docker kill $(IM) || echo not running ;
 	docker rm $(IM) || echo not made 
 
-docker-publish-no-build:
-	@docker push $(IM):$(VERSION) \
-	&& docker push $(IM):latest
-
-docker-publish-dev-no-build:
-	@docker push $(DEV):$(VERSION) \
-	&& docker push $(DEV):latest
-
-docker-publish: docker-build
-	@docker push $(IM):$(VERSION) \
-	&& docker push $(IM):latest
+#### TESTING #####
 
-docker-test: docker-build
+docker-test-full: docker-build
 	docker images | grep odkfull &&\
 	make test CMD=./seed-via-docker.sh
 
+docker-test-dev: docker-build-dev
+	docker images | grep odkdev &&\
+	make test CMD=./seed-via-docker.sh
+
+docker-test-lite: docker-build
+	docker images | grep odklite &&\
+	make test CMD=./seed-via-docker.sh
+
+docker-test: docker-test-full docker-test-dev
+
 docker-test-no-build:
 	docker images | grep odkfull &&\
 	make test CMD=./seed-via-docker.sh
 
 docker-test-dev-no-build:
 	docker images | grep odkdev &&\
 	make test CMD=./seed-via-docker.sh
+
+docker-test-lite-no-build:
+	docker images | grep odklite &&\
+	make test CMD=./seed-via-docker.sh
+
+#### Publishing #####
+
+docker-publish-no-build:
+	@docker push $(DEV):$(VERSION) \
+	&& docker push $(DEV):latest \
+	&& docker push $(IMLITE):latest \
+	&& docker push $(IMLITE):$(VERSION) \
+	&& docker push $(IM):latest \
+	&& docker push $(IM):$(VERSION)
+
+docker-publish: docker-build
+	@docker push $(DEV):$(VERSION) \
+	&& docker push $(DEV):latest \
+	&& docker push $(IMLITE):latest \
+	&& docker push $(IMLITE):$(VERSION) \
+	&& docker push $(IM):latest \
+	&& docker push $(IM):$(VERSION)
 
-docker-publish-all: docker-publish
-	(cd docker/osklite && make publish VERSION=$(VERSION))
+docker-publish-dev-no-build:
+	@docker push $(DEV):$(VERSION) \
+	&& docker push $(DEV):latest
 
 clean-tests:
 	rm -rf target/*
diff --git a/docker/odklite/Dockerfile b/docker/odklite/Dockerfile
@@ -1,22 +1,66 @@
-FROM openjdk:8-jdk-alpine
-MAINTAINER Chris Mungall <[email protected]>
-
-RUN apk add --update make
+### From https://stackoverflow.com/questions/51121875/how-to-run-docker-with-python-and-java
+### 1. Get Linux
+FROM ubuntu:18.04
+LABEL maintainer="[email protected]" 
 
+### 2. Get Java, Python and all required system libraries (version control etc)
+ENV JAVA_HOME="/usr"
 WORKDIR /tools
+ENV PATH "/tools/:$PATH"
+COPY requirements.txt.lite /tools/requirements.txt
+COPY odk/make-release-assets.py /tools/
 
+#ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
+# LAYERSIZE ~1000MB
+RUN apt-get update &&\
+  apt-get install -y software-properties-common &&\
+  apt-get upgrade -y &&\
+  apt-get install -y build-essential \
+    git \
+    openjdk-8-jre \
+    make \
+    unzip \
+    rsync \
+    curl \
+    jq \
+    openssl
+
+###### owltools & OORT ######
 # For now we get these from jenkins builds, but these should be obtained
 # by composing existing Dockerfiles, or by obtaining directly from maven
-RUN wget http://build.berkeleybop.org/job/robot/lastSuccessfulBuild/artifact/bin/robot -O /tools/robot
-RUN wget http://build.berkeleybop.org/job/robot/lastSuccessfulBuild/artifact/bin/robot.jar -O /tools/robot.jar
-RUN chmod +x /tools/*
-ENV PATH "/tools/:$PATH"
+ENV OWLTOOLS 2020-04-06
+RUN curl -L https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS/owltools -o /tools/owltools && \
+    curl -L https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS/ontology-release-runner -o /tools/ontology-release-runner && \
+    curl -L https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS/owltools-oort-all.jar -o /tools/owltools-oort-all.jar && \
+    chmod +x /tools/owltools && \
+    chmod +x /tools/ontology-release-runner && \
+    chmod +x /tools/owltools-oort-all.jar
+
+###### ROBOT ######
+ENV ROBOT v1.8.1
+ARG ROBOT_JAR=https://github.com/ontodev/robot/releases/download/$ROBOT/robot.jar
+ENV ROBOT_JAR ${ROBOT_JAR}
+
+# LAYERSIZE ~66MB
+RUN curl -L $ROBOT_JAR -o /tools/robot.jar &&\
+    curl -L https://raw.githubusercontent.com/ontodev/robot/$ROBOT/bin/robot -o /tools/robot && \
+    chmod +x /tools/robot && \
+    chmod +x /tools/robot.jar
+
+### 5. Install ODK
+ARG ODK_VERSION=0.0.0
+ENV ODK_VERSION=${ODK_VERSION}
 
-# Setup dosdp tools
-ENV V=0.10.1
-RUN wget -nv  https://github.com/INCATools/dosdp-tools/releases/download/v$V/dosdp-tools-$V.tgz
-RUN tar -zxvf dosdp-tools-$V.tgz && mv dosdp-tools-$V /tools/dosdp-tools
-ENV PATH "/tools/dosdp-tools/bin:$PATH"
+### TODO REVIEW THIS. As we speak, jq is official still stalled at 1.5, but for the walk function, we
+### Need 1.6 - once 1.6 is offcial, the following RUN can be removed.
+# LAYERSIZE: ~4MB
+RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /tools/jq && chmod +x /tools/jq
 
-CMD make -h
+COPY odk/odk.py /tools/
+COPY template/ /tools/templates/
+RUN chmod +x /tools/*.py
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+RUN echo "Keep me in sync with obolibrary/odkfull!"
 
+CMD python /tools/odk.py
diff --git a/docker/odklite/Makefile b/docker/odklite/Makefile
@@ -1,14 +1,19 @@
 
 # Building docker image
-VERSION = "v1.1.7" 
+VERSION = "v1.2.27" 
 IM=obolibrary/odklite
 
 build:
 	@docker build -t $(IM):$(VERSION) . \
 	&& docker tag $(IM):$(VERSION) $(IM):latest
 
-run:
-	docker run --rm -ti --name odklite $(IM)
+docker-build-no-cache:
+	@docker build  --build-arg ODK_VERSION=$(VERSION) --no-cache -t $(IM):$(VERSION) . \
+	&& docker tag $(IM):$(VERSION) $(IM):latest
+
+docker-build:
+	@docker build --build-arg ODK_VERSION=$(VERSION) -t $(IM):$(VERSION) . \
+	&& docker tag $(IM):$(VERSION) $(IM):latest
 
 publish: build
 	@docker push $(IM):$(VERSION) \

diff --git a/docker/robot/Dockerfile b/docker/robot/Dockerfile
@@ -1,17 +1,24 @@
-FROM openjdk:8-jre-alpine3.9
-MAINTAINER Chris Mungall <[email protected]>
-
-RUN apk add --no-cache make bash
+FROM openjdk:8-slim
+LABEL maintainer="[email protected]" 
 
 WORKDIR /tools
 
-ENV ROBOT v1.5.0
+ENV ROBOT v1.8.1
 ARG ROBOT_JAR=https://github.com/ontodev/robot/releases/download/$ROBOT/robot.jar
 ENV ROBOT_JAR ${ROBOT_JAR}
 
-RUN wget $ROBOT_JAR -O /tools/robot.jar && \
-    wget https://raw.githubusercontent.com/ontodev/robot/$ROBOT/bin/robot -O /tools/robot && \
-    chmod +x /tools/*
+RUN apt-get update &&\
+  apt-get upgrade -y &&\
+  apt-get install -y make \
+    unzip \
+    rsync \
+    curl
+
+RUN curl -L $ROBOT_JAR -o /tools/robot.jar &&\
+    curl -L https://raw.githubusercontent.com/ontodev/robot/$ROBOT/bin/robot -o /tools/robot && \
+    chmod +x /tools/robot && \
+    chmod +x /tools/robot.jar
 
 ENV PATH "/tools/:$PATH"
+RUN echo "Keep me in sync with obolibrary/odkfull!"
 CMD robot -h
diff --git a/docker/robot/Makefile b/docker/robot/Makefile
@@ -1,15 +1,12 @@
 
 # Building docker image
-VERSION = "v1.5.0" 
+VERSION = "v1.8.1" 
 IM=obolibrary/robot
 
-build:
+docker-build:
 	@docker build -t $(IM):$(VERSION) . \
 	&& docker tag $(IM):$(VERSION) $(IM):latest
 
-run:
-	docker run --rm -ti --name robot $(IM)
-
 clean:
 	docker kill $(IM) || echo not running ;
 	docker rm $(IM) || echo not made 

diff --git a/docker/testdocker/Dockerfile b/docker/testdocker/Dockerfile