Skip to content

Commit 15ae005

Browse files
vitalykorolevvitalykorolev
authored
Merge pull request #269 from marklogic/CLD-660-CLD-617-rootless
Merge rootless feature branch to develop
2 parents 118b81e + 2dbf0ec commit 15ae005

14 files changed

+1476
-533
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,4 @@ test/test_results/*
2727
test/log.html
2828
test/report.html
2929
test/output.xml
30+
test/python_env/*

‎Jenkinsfile

+44-34
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ void copyRPMs() {
136136
error "Invalid value in marklogicVersion parameter."
137137
}
138138
sh """
139-
cd src/centos
139+
cd src
140140
if [ -z ${env.ML_RPM} ]; then
141141
wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/server/MarkLogic-${RPMversion}${RPMsuffix}.x86_64.rpm
142142
else
@@ -149,43 +149,47 @@ void copyRPMs() {
149149
fi
150150
"""
151151
script {
152-
RPM = sh(returnStdout: true, script: 'cd src/centos;file MarkLogic-*.rpm | cut -d: -f1').trim()
153-
CONVERTERS = sh(returnStdout: true, script: 'cd src/centos;file MarkLogicConverters-*.rpm | cut -d: -f1').trim()
152+
RPM = sh(returnStdout: true, script: 'cd src;file MarkLogic-*.rpm | cut -d: -f1').trim()
153+
CONVERTERS = sh(returnStdout: true, script: 'cd src;file MarkLogicConverters-*.rpm | cut -d: -f1').trim()
154154
mlVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim()
155155
}
156156
}
157157

158+
void buildDockerImage() {
159+
sh "make build docker_image_type=${dockerImageType} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
160+
currentBuild.displayName = "#${BUILD_NUMBER} ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
161+
}
162+
158163
void structureTests() {
159164
sh """
160-
cd test
161-
#insert current version
162-
sed -i -e 's^VERSION_PLACEHOLDER^${mlVersion}-${env.platformString}-${env.dockerVersion}^g' -e 's^BRANCH_PLACEHOLDER^${env.BRANCH_NAME}^g' ./structure-test.yaml
163-
cd ..
164-
curl -s -LO https://storage.googleapis.com/container-structure-test/v1.11.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
165-
make structure-test version=${mlVersion}-${env.platformString}-${env.dockerVersion} Jenkins=true
166-
#fix junit output
167-
sed -i -e 's/<\\/testsuites>//' -e 's/<testsuite>//' -e 's/<testsuites/<testsuite name="container-structure-test"/' ./container-structure-test.xml
165+
#install container-structure-test 1.16.0 binary
166+
curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
167+
make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true
168168
"""
169169
}
170170

171+
void dockerTests() {
172+
sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${dockerImageType}"
173+
}
174+
171175
void lint() {
172-
IMAGE_INFO = sh(returnStdout: true, script: 'docker images | grep \"marklogic-server-centos\"')
176+
IMAGE_INFO = sh(returnStdout: true, script: 'docker images | grep \"marklogic-server-'+"${dockerImageType}"+'\"')
173177

174-
sh '''
178+
sh """
175179
make lint Jenkins=true
176-
cat start-marklogic-lint.txt marklogic-deps-centos-base-lint.txt marklogic-server-centos-base-lint.txt
177-
'''
180+
cat start-scripts-lint.txt dockerfile-lint.txt
181+
"""
178182

179-
LINT_OUTPUT = sh(returnStdout: true, script: 'echo start-marklogic.sh: ;echo; cat start-marklogic-lint.txt; echo dockerfile-marklogic-server-centos: ; echo marklogic-deps-centos:base: ;echo; cat marklogic-deps-centos-base-lint.txt; echo marklogic-server-centos:base: ;echo; cat marklogic-server-centos-base-lint.txt').trim()
183+
LINT_OUTPUT = sh(returnStdout: true, script: "echo start-scripts-lint.txt: ;echo; cat start-scripts-lint.txt; echo; echo dockerfile-lint.txt: ; cat dockerfile-lint.txt; echo").trim()
180184

181-
sh '''
182-
rm -f start-marklogic-lint.txt marklogic-deps-centos-base-lint.txt marklogic-server-centos-base-lint.txt
183-
'''
185+
sh """
186+
rm -f start-scripts-lint.txt dockerfile-lint.txt
187+
"""
184188
}
185189

186-
void scan() {
190+
void vulnerabilityScan() {
187191
sh """
188-
make scan version=${mlVersion}-${env.platformString}-${env.dockerVersion} Jenkins=true
192+
make scan current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
189193
grep \'High\\|Critical\' scan-server-image.txt
190194
"""
191195

@@ -198,11 +202,16 @@ void scan() {
198202
}
199203

200204
void publishToInternalRegistry() {
201-
publishTag="${mlVersion}-${env.platformString}-${env.dockerVersion}"
205+
currentImage="marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
206+
mlVerShort=mlVersion.split("\\.")[0]
207+
latestTag="marklogic/marklogic-server-${dockerImageType}:latest-${mlVerShort}"
202208
withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) {
203209
sh """
204210
echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${dockerRegistry}
205-
make push-mlregistry version=${publishTag}
211+
docker tag ${currentImage} ${dockerRegistry}/${currentImage}
212+
docker tag ${currentImage} ${dockerRegistry}/${latestTag}
213+
docker push ${dockerRegistry}/${currentImage}
214+
docker push ${dockerRegistry}/${latestTag}
206215
"""
207216

208217
}
@@ -216,13 +225,13 @@ void publishToInternalRegistry() {
216225
]]) {
217226
sh """
218227
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com
219-
docker tag local-dev/marklogic-server-centos:${publishTag} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${publishTag}
220-
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${publishTag}
228+
docker tag ${currentImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}
229+
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}
221230
"""
222231
}
223232
}
224233

225-
currentBuild.description = "Publish ${publishTag}"
234+
currentBuild.description = "Published"
226235
}
227236

228237
void publishTestResults() {
@@ -242,9 +251,10 @@ pipeline {
242251
skipStagesAfterUnstable()
243252
}
244253
triggers {
245-
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=10
246-
00 04 * * * % marklogicVersion=11
247-
00 05 * * * % marklogicVersion=12''' : '')
254+
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=10 dockerImageType=centos
255+
00 04 * * * % marklogicVersion=11 dockerImageType=centos
256+
00 05 * * * % marklogicVersion=12 dockerImageType=centos
257+
00 06 * * * % marklogicVersion=11 dockerImageType=ubi''' : '')
248258
}
249259
environment {
250260
QA_LICENSE_KEY = credentials('QA_LICENSE_KEY')
@@ -253,7 +263,7 @@ pipeline {
253263
parameters {
254264
string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
255265
string(name: 'dockerVersion', defaultValue: '1.1.2', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
256-
string(name: 'platformString', defaultValue: 'centos', description: 'Platform string for Docker image version. Will be made part of the docker image tag', trim: true)
266+
choice(name: 'dockerImageType', choices: 'centos\nubi\nubi-rootless', description: 'Platform type for Docker image. Will be made part of the docker image tag')
257267
choice(name: 'marklogicVersion', choices: '11\n12\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')
258268
string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true)
259269
string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true)
@@ -277,7 +287,7 @@ pipeline {
277287

278288
stage('Build-Image') {
279289
steps {
280-
sh "make build version=${mlVersion}-${env.platformString}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
290+
buildDockerImage()
281291
}
282292
}
283293

@@ -289,7 +299,7 @@ pipeline {
289299

290300
stage('Scan') {
291301
steps {
292-
scan()
302+
vulnerabilityScan()
293303
}
294304
}
295305

@@ -307,7 +317,7 @@ pipeline {
307317
expression { return params.DOCKER_TESTS }
308318
}
309319
steps {
310-
sh "make docker-tests test_image=local-dev/marklogic-server-centos:${mlVersion}-${env.platformString}-${env.dockerVersion} version=${mlVersion}-${env.platformString}-${env.dockerVersion} build_branch=${env.BRANCH_NAME}"
320+
dockerTests()
311321
}
312322
}
313323

@@ -327,7 +337,7 @@ pipeline {
327337
post {
328338
always {
329339
sh '''
330-
cd src/centos
340+
cd src
331341
rm -rf *.rpm
332342
docker rm -f $(docker ps -a -q) || true
333343
docker system prune --force --filter "until=720h"

‎Makefile

+33-23
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,74 @@
11
version?=internal
22
package?=MarkLogic.rpm
3-
REPONAME=local-dev
4-
docker_registry?=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com
5-
repoDir=marklogic
3+
repo_dir=marklogic
64
docker_build_options=--compress
7-
test_image?=${docker_registry}/${repoDir}/marklogic-server-centos:${version}
85
build_branch?=local
6+
docker_image_type?=ubi
7+
current_image?=${repo_dir}/marklogic-server-${docker_image_type}:${version}
98

109
#***************************************************************************
11-
# build docker images
10+
# build docker image
1211
#***************************************************************************
1312
build:
14-
cd src/centos/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-deps-centos:${version}" -f ../../dockerFiles/marklogic-deps-centos:base .
15-
cd src/centos/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-server-centos:${version}" --build-arg BASE_IMAGE=${REPONAME}/marklogic-deps-centos:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../../dockerFiles/marklogic-server-centos:base .
16-
13+
#ubi-rootless uses the same dependencies as ubi image
14+
ifeq ($(docker_image_type),ubi-rootless)
15+
cp dockerFiles/marklogic-deps-ubi\:base dockerFiles/marklogic-deps-ubi-rootless\:base
16+
endif
17+
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-deps-${docker_image_type}:${version}" -f ../dockerFiles/marklogic-deps-${docker_image_type}:base .
18+
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-server-${docker_image_type}:${version}" --build-arg BASE_IMAGE=${repo_dir}/marklogic-deps-${docker_image_type}:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-${docker_image_type}:base .
19+
rm -f dockerFiles/marklogic-deps-ubi-rootless\:base
1720
#***************************************************************************
1821
# strcture test docker images
1922
#***************************************************************************
2023
structure-test:
21-
container-structure-test test --config ./test/structure-test.yaml --image ${REPONAME}/marklogic-server-centos:${version} \
24+
ifeq ($(docker_image_type),ubi-rootless)
25+
@echo type is ${docker_image_type}
26+
sed -i -e 's^DOCKER_PID_PLACEHOLDER^/home/marklogic_user/MarkLogic.pid^g' ./test/structure-test.yaml
27+
else
28+
@echo type is ${docker_image_type}
29+
sed -i -e 's^DOCKER_PID_PLACEHOLDER^/var/run/MarkLogic.pid^g' ./test/structure-test.yaml
30+
endif
31+
sed -i -e 's^VERSION_PLACEHOLDER^${version}^g' ./test/structure-test.yaml
32+
sed -i -e 's^BRANCH_PLACEHOLDER^${build_branch}^g' ./test/structure-test.yaml
33+
container-structure-test test --config ./test/structure-test.yaml --image ${current_image} \
2234
$(if $(Jenkins), --output junit | tee container-structure-test.xml,)
2335

2436
#***************************************************************************
2537
# docker image tests
2638
#***************************************************************************
2739
docker-tests:
2840
cd test; python3 -m venv python_env
29-
cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --variable TEST_IMAGE:${test_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --maxerrorlines 9999 ./docker-tests.robot; deactivate
41+
cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 ./docker-tests.robot; deactivate
3042
rm -r test/python_env/
3143

3244
#***************************************************************************
3345
# run all tests
3446
#***************************************************************************
3547
.PHONY: test
3648
test: structure-test docker-tests
37-
38-
#***************************************************************************
39-
# push docker images to mlregistry.marklogic.com
40-
#***************************************************************************
41-
push-mlregistry:
42-
docker tag ${REPONAME}/marklogic-server-centos:${version} ${docker_registry}/${repoDir}/marklogic-server-centos:${version}
43-
docker push ${docker_registry}/${repoDir}/marklogic-server-centos:${version}
4449

4550
#***************************************************************************
46-
# run lint checker on Dockerfiles, print linting issues but do not fail the build
51+
# run lint checker on shell scripts and Dockerfiles, print linting issues but do not fail the build
4752
#***************************************************************************
53+
4854
lint:
49-
docker run --rm -v "${PWD}:/mnt" koalaman/shellcheck:stable src/centos/scripts/start-marklogic.sh $(if $(Jenkins), > start-marklogic-lint.txt,)
50-
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/marklogic-deps-centos:base $(if $(Jenkins), > marklogic-deps-centos-base-lint.txt,)
51-
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/marklogic-server-centos:base $(if $(Jenkins), > marklogic-server-centos-base-lint.txt,)
55+
docker run --rm -v "${PWD}:/mnt" koalaman/shellcheck:stable src/scripts/*.sh $(if $(Jenkins), > start-scripts-lint.txt,)
56+
57+
@for dockerFile in $(shell ls ./dockerFiles/); do\
58+
echo "Lint results for $${dockerFile}" $(if $(Jenkins), >> dockerfile-lint.txt,) ; \
59+
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/$${dockerFile} $(if $(Jenkins), >> dockerfile-lint.txt,);\
60+
echo $(if $(Jenkins), >> dockerfile-lint.txt,) ;\
61+
done
5262

5363
#***************************************************************************
5464
# security scan docker images
5565
#***************************************************************************
5666
scan:
57-
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock anchore/grype:latest ${REPONAME}/marklogic-server-centos:${version} $(if $(Jenkins), > scan-server-image.txt,)
67+
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock anchore/grype:latest ${current_image} $(if $(Jenkins), > scan-server-image.txt,)
5868

5969
#***************************************************************************
6070
# remove junk
6171
#***************************************************************************
6272
clean:
6373
rm -f *.log
64-
rm -f *.rpm
74+
rm -f *.rpm

‎dockerFiles/marklogic-deps-centos:base

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ FROM centos:centos7
88
LABEL "com.marklogic.maintainer"="[email protected]"
99

1010
###############################################################
11-
# install networking, base deps and ntp/tzdata for timezone
11+
# install networking, base deps and tzdata for timezone
1212
###############################################################
1313

1414
RUN echo "NETWORKING=yes" > /etc/sysconfig/network \

‎dockerFiles/marklogic-deps-ubi:base

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
###############################################################
2+
#
3+
# Copyright (c) 2023 MarkLogic Corporation
4+
#
5+
###############################################################
6+
7+
FROM registry.access.redhat.com/ubi8/ubi-minimal
8+
LABEL "com.marklogic.maintainer"="[email protected]"
9+
10+
###############################################################
11+
# install libnsl rpm package
12+
###############################################################
13+
14+
RUN microdnf install -y wget \
15+
&& wget --quiet https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-release-tierpoint/devdependencies/libnsl-2.28-236.el8_9.12.x86_64.rpm \
16+
&& rpm -i libnsl-2.28-236.el8_9.12.x86_64.rpm \
17+
&& rm libnsl-2.28-236.el8_9.12.x86_64.rpm
18+
19+
###############################################################
20+
# install networking, base deps and tzdata for timezone
21+
###############################################################
22+
23+
RUN echo "NETWORKING=yes" > /etc/sysconfig/network \
24+
&& microdnf -y update \
25+
&& microdnf -y install gdb.x86_64 glibc.i686 libstdc++.i686 libgcc.i686 initscripts redhat-lsb-core.x86_64 tzdata \
26+
&& microdnf clean all
27+
28+
###############################################################
29+
# platform configurations
30+
###############################################################

0 commit comments

Comments
 (0)