MLE-11923 update docker tests (#246)

vitalykorolev · web-flow · commit 534b1c0942e3 · 2024-02-06T10:53:52.000-08:00
* Add condition to skip tests that are not applicable to rootless
* update params and add image_type
* Update pipeline to handle new test parameters
* update .gitignore to skip python_env
* add ignore yum version req to docker linter
* remove sudo requirement for CST install
* switch docker ver to ML ver for structure tests
* downgrade hadolint warning and add dnf to it
* fix ML version for structure test.. again.
* fix hadlint rules
* Added tests to verify timezone change in the container. change tab zize to 4 for readability.
* randomize docker test execution
* add image type to build number
* remove unused links and add tzdata reinstall
* remove extra space
* Update steps to verify time zone via REST
* add a note about rootless image
* add timezone set to ubi image
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,4 @@ test/test_results/*
 test/log.html
 test/report.html
 test/output.xml
+test/python_env/*
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -157,24 +157,19 @@ void copyRPMs() {
 
 void buildDockerImage() {
     sh "make build docker_image_type=${dockerImageType} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
+    currentBuild.displayName = "#${BUILD_NUMBER} ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
 }
 
 void structureTests() {
     sh """
-        cd test
-        #insert current version
-        sed -i -e 's^VERSION_PLACEHOLDER^${mlVersion}-${env.dockerImageType}-${env.dockerVersion}^g' -e 's^BRANCH_PLACEHOLDER^${env.BRANCH_NAME}^g' ./structure-test.yaml
-        cd ..
-        #install container-structure-test binary
-        curl -s -LO https://storage.googleapis.com/container-structure-test/v1.11.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
-        make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
-        #fix junit output
-        sed -i -e 's/<\\/testsuites>//' -e 's/<testsuite>//' -e 's/<testsuites/<testsuite name="container-structure-test"/' ./container-structure-test.xml
+        #install container-structure-test 1.16.0 binary
+        curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
+        make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true
     """
 }
 
 void dockerTests() {
-    sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME}"
+    sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${dockerImageType}"
 }
 
 void lint() {
@@ -232,7 +227,7 @@ void publishToInternalRegistry() {
             }
     }
 
-    currentBuild.description = "Publish ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
+    currentBuild.description = "Published"
 }
 
 void publishTestResults() {
diff --git a/Makefile b/Makefile
@@ -21,6 +21,15 @@ endif
 # strcture test docker images
 #***************************************************************************
 structure-test:
+ifeq ($(docker_image_type),ubi-rootless)
+	@echo type is ${docker_image_type}
+	sed -i -e 's^DOCKER_PID_PLACEHOLDER^/home/marklogic_user/MarkLogic.pid^g' ./test/structure-test.yaml
+else
+	@echo type is ${docker_image_type}
+	sed -i -e 's^DOCKER_PID_PLACEHOLDER^/var/run/MarkLogic.pid^g' ./test/structure-test.yaml
+endif
+	sed -i -e 's^VERSION_PLACEHOLDER^${version}^g' ./test/structure-test.yaml
+	sed -i -e 's^BRANCH_PLACEHOLDER^${build_branch}^g' ./test/structure-test.yaml
 	container-structure-test test --config ./test/structure-test.yaml --image ${current_image} \
 		$(if $(Jenkins), --output junit | tee container-structure-test.xml,)
 
@@ -29,7 +38,7 @@ structure-test:
 #***************************************************************************
 docker-tests: 
 	cd test; python3 -m venv python_env
-	cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --variable TEST_IMAGE:${current_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --maxerrorlines 9999 ./docker-tests.robot; deactivate
+	cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 ./docker-tests.robot; deactivate
 	rm -r test/python_env/
 	
 #***************************************************************************
diff --git a/dockerFiles/marklogic-server-ubi-rootless:base b/dockerFiles/marklogic-server-ubi-rootless:base
@@ -123,8 +123,8 @@ ENV MARKLOGIC_INSTALL_DIR=/opt/MarkLogic  \
 ################################################################
 # Set Timezone
 ################################################################
-RUN microdnf reinstall -y tzdata \
-    && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN microdnf -y reinstall tzdata
 
 ################################################################
 # Set appropriate authorisation to MARKLOGIC_DATA_DIR 
diff --git a/dockerFiles/marklogic-server-ubi:base b/dockerFiles/marklogic-server-ubi:base
@@ -98,6 +98,11 @@ ENV MARKLOGIC_INSTALL_DIR=/opt/MarkLogic  \
     BUILD_BRANCH=${BUILD_BRANCH} \
     MARKLOGIC_JOIN_TLS_ENABLED=false
     
+################################################################
+# Set Timezone
+################################################################
+
+RUN microdnf -y reinstall tzdata
 
 ###############################################################
 # expose MarkLogic server ports
diff --git a/hadolint.yaml b/hadolint.yaml
@@ -1 +1,4 @@
 failure-threshold: error
+ignored:
+  - DL3033
+  - DL3041
diff --git a/src/scripts/start-marklogic-rootless.sh b/src/scripts/start-marklogic-rootless.sh
@@ -64,7 +64,7 @@ fi
 
     [[ "${MARKLOGIC_PID_FILE}" ]] && echo "export MARKLOGIC_PID_FILE=$MARKLOGIC_PID_FILE" >>/etc/marklogic.conf
     [[ "${MARKLOGIC_UMASK}" ]] && echo "export MARKLOGIC_UMASK=$MARKLOGIC_UMASK" >>/etc/marklogic.conf
-    [[ "${TZ}" ]] && echo "export TZ=$TZ " >>/etc/marklogic.conf
+    [[ "${TZ}" ]] && echo "export TZ=$TZ" >>/etc/marklogic.conf
     [[ "${ML_HUGEPAGES_TOTAL}" ]] && echo "export ML_HUGEPAGES_TOTAL=$ML_HUGEPAGES_TOTAL" >>/etc/marklogic.conf
     [[ "${MARKLOGIC_DISABLE_JVM}" ]] && echo "export MARKLOGIC_DISABLE_JVM=$MARKLOGIC_DISABLE_JVM" >>/etc/marklogic.conf
     [[ "${MARKLOGIC_USER}" ]] && echo "export MARKLOGIC_USER=$MARKLOGIC_USER" >>/etc/marklogic.conf
diff --git a/test/README.md b/test/README.md
@@ -34,7 +34,10 @@ In order to run all tests you can use make from root folder with
 or by running Robot in test directly with 
 `robot ./docker-tests.robot`
 
+*Note*
 QA_LICENSE_KEY environment variable needs to be set to a valid license key for license test to pass.
+If UBI rootless image is used, specify image type for individual tests:
+`robot --variable TEST_IMAGE:marklogic/marklogic-server-ubi-rootless:internal --variable IMAGE_TYPE:ubi-rootless --test 'Initialized MarkLogic container with config overrides' docker-tests.robot`
 
 For a quick start guide for Robot framework see https://robotframework.org/#getting-started
 Full user guide is available at https://robotframework.org/robotframework/#user-guide
diff --git a/test/docker-tests.robot b/test/docker-tests.robot
diff --git a/test/keywords.resource b/test/keywords.resource
diff --git a/test/structure-test.yaml b/test/structure-test.yaml

Original file line number	Diff line number	Diff line change
`@@ -157,24 +157,19 @@ void copyRPMs() {`
`157`	`157`
`158`	`158`	`void buildDockerImage() {`
`159`	`159`	`sh "make build docker_image_type=${dockerImageType} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"`
	`160`	`+ currentBuild.displayName = "#${BUILD_NUMBER} ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"`
`160`	`161`	`}`
`161`	`162`
`162`	`163`	`void structureTests() {`
`163`	`164`	`sh """`
`164`		`- cd test`
`165`		`- #insert current version`
`166`		`- sed -i -e 's^VERSION_PLACEHOLDER^${mlVersion}-${env.dockerImageType}-${env.dockerVersion}^g' -e 's^BRANCH_PLACEHOLDER^${env.BRANCH_NAME}^g' ./structure-test.yaml`
`167`		`- cd ..`
`168`		`- #install container-structure-test binary`
`169`		`- curl -s -LO https://storage.googleapis.com/container-structure-test/v1.11.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test`
`170`		`- make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true`
`171`		`- #fix junit output`
`172`		`- sed -i -e 's/<\\/testsuites>//' -e 's/<testsuite>//' -e 's/<testsuites/<testsuite name="container-structure-test"/' ./container-structure-test.xml`
	`165`	`+ #install container-structure-test 1.16.0 binary`
	`166`	`+ curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test`
	`167`	`+ make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true`
`173`	`168`	`"""`
`174`	`169`	`}`
`175`	`170`
`176`	`171`	`void dockerTests() {`
`177`		`- sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME}"`
	`172`	`+ sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${dockerImageType}"`
`178`	`173`	`}`
`179`	`174`
`180`	`175`	`void lint() {`
`@@ -232,7 +227,7 @@ void publishToInternalRegistry() {`
`232`	`227`	`}`
`233`	`228`	`}`
`234`	`229`
`235`		`- currentBuild.description = "Publish ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"`
	`230`	`+ currentBuild.description = "Published"`
`236`	`231`	`}`
`237`	`232`
`238`	`233`	`void publishTestResults() {`