Skip to content

Commit b5188a9

Browse files
rwinieskirwinieski
committed
Merge branch 'CLD-660-CLD-617-rootless' of https://github.com/marklogic/marklogic-docker into CLD-660-CLD-617-rootless
2 parents 0f2c845 + 41b5106 commit b5188a9

File tree

4 files changed

+65
-70
lines changed

4 files changed

+65
-70
lines changed

‎Jenkinsfile

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import groovy.json.JsonSlurperClassic
99
emailList = '[email protected], [email protected], [email protected], [email protected], [email protected]'
1010
// email list for security vulnerabilities only
1111
emailSecList = '[email protected], [email protected]'
12-
gitCredID = 'marklogic-builder'
12+
gitCredID = 'marklogic-builder-github'
1313
dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com'
14-
JIRA_ID_PATTERN = /(?i)(CLD|DEVO|QAINF|BUG|DBI)-\d{3,4}/
14+
JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/
1515
JIRA_ID = ''
1616
LINT_OUTPUT = ''
1717
SCAN_OUTPUT = ''
@@ -155,37 +155,46 @@ void copyRPMs() {
155155
}
156156
}
157157

158+
void buildDockerImage() {
159+
sh "make build docker_image_type=${dockerImageType} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
160+
}
161+
158162
void structureTests() {
159163
sh """
160164
cd test
161165
#insert current version
162-
sed -i -e 's^VERSION_PLACEHOLDER^${mlVersion}-${env.platformString}-${env.dockerVersion}^g' -e 's^BRANCH_PLACEHOLDER^${env.BRANCH_NAME}^g' ./structure-test.yaml
166+
sed -i -e 's^VERSION_PLACEHOLDER^${mlVersion}-${env.dockerImageType}-${env.dockerVersion}^g' -e 's^BRANCH_PLACEHOLDER^${env.BRANCH_NAME}^g' ./structure-test.yaml
163167
cd ..
168+
#install container-structure-test binary
164169
curl -s -LO https://storage.googleapis.com/container-structure-test/v1.11.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
165-
make structure-test version=${mlVersion}-${env.platformString}-${env.dockerVersion} Jenkins=true
170+
make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
166171
#fix junit output
167172
sed -i -e 's/<\\/testsuites>//' -e 's/<testsuite>//' -e 's/<testsuites/<testsuite name="container-structure-test"/' ./container-structure-test.xml
168173
"""
169174
}
170175

176+
void dockerTests() {
177+
sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME}"
178+
}
179+
171180
void lint() {
172-
IMAGE_INFO = sh(returnStdout: true, script: 'docker images | grep \"marklogic-server-centos\"')
181+
IMAGE_INFO = sh(returnStdout: true, script: 'docker images | grep \"marklogic-server-'+"${dockerImageType}"+'\"')
173182

174-
sh '''
183+
sh """
175184
make lint Jenkins=true
176-
cat start-marklogic-lint.txt marklogic-deps-centos-base-lint.txt marklogic-server-centos-base-lint.txt
177-
'''
185+
cat start-scripts-lint.txt dockerfile-lint.txt
186+
"""
178187

179-
LINT_OUTPUT = sh(returnStdout: true, script: 'echo start-marklogic.sh: ;echo; cat start-marklogic-lint.txt; echo dockerfile-marklogic-server-centos: ; echo marklogic-deps-centos:base: ;echo; cat marklogic-deps-centos-base-lint.txt; echo marklogic-server-centos:base: ;echo; cat marklogic-server-centos-base-lint.txt').trim()
188+
LINT_OUTPUT = sh(returnStdout: true, script: "echo start-scripts-lint.txt: ;echo; cat start-scripts-lint.txt; echo; echo dockerfile-lint.txt: ; cat dockerfile-lint.txt; echo").trim()
180189

181-
sh '''
182-
rm -f start-marklogic-lint.txt marklogic-deps-centos-base-lint.txt marklogic-server-centos-base-lint.txt
183-
'''
190+
sh """
191+
rm -f start-scripts-lint.txt dockerfile-lint.txt
192+
"""
184193
}
185194

186-
void scan() {
195+
void vulnerabilityScan() {
187196
sh """
188-
make scan version=${mlVersion}-${env.platformString}-${env.dockerVersion} Jenkins=true
197+
make scan current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
189198
grep \'High\\|Critical\' scan-server-image.txt
190199
"""
191200

@@ -198,11 +207,12 @@ void scan() {
198207
}
199208

200209
void publishToInternalRegistry() {
201-
publishTag="${mlVersion}-${env.platformString}-${env.dockerVersion}"
210+
currentImage="marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
202211
withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) {
203212
sh """
204213
echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${dockerRegistry}
205-
make push-mlregistry version=${publishTag}
214+
docker tag ${currentImage} ${dockerRegistry}/${currentImage}
215+
docker push ${dockerRegistry}/${currentImage}
206216
"""
207217

208218
}
@@ -215,14 +225,14 @@ void publishToInternalRegistry() {
215225
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
216226
]]) {
217227
sh """
218-
aws ecr get-login --no-include-email --region us-west-2 | bash
219-
docker tag local-dev/marklogic-server-centos:${publishTag} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${publishTag}
220-
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${publishTag}
228+
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com
229+
docker tag ${currentImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}
230+
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}
221231
"""
222232
}
223233
}
224234

225-
currentBuild.description = "Publish ${publishTag}"
235+
currentBuild.description = "Publish ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
226236
}
227237

228238
void publishTestResults() {
@@ -242,9 +252,10 @@ pipeline {
242252
skipStagesAfterUnstable()
243253
}
244254
triggers {
245-
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=10
246-
00 04 * * * % marklogicVersion=11
247-
00 05 * * * % marklogicVersion=12''' : '')
255+
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=10 dockerImageType=centos
256+
00 04 * * * % marklogicVersion=11 dockerImageType=centos
257+
00 05 * * * % marklogicVersion=12 dockerImageType=centos
258+
00 06 * * * % marklogicVersion=11 dockerImageType=ubi''' : '')
248259
}
249260
environment {
250261
QA_LICENSE_KEY = credentials('QA_LICENSE_KEY')
@@ -253,7 +264,7 @@ pipeline {
253264
parameters {
254265
string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
255266
string(name: 'dockerVersion', defaultValue: '1.1.1', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
256-
string(name: 'platformString', defaultValue: 'centos', description: 'Platform string for Docker image version. Will be made part of the docker image tag', trim: true)
267+
choice(name: 'dockerImageType', choices: 'centos\nubi\nubi-rootless', description: 'Platform type for Docker image. Will be made part of the docker image tag')
257268
choice(name: 'marklogicVersion', choices: '11\n12\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')
258269
string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true)
259270
string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true)
@@ -277,7 +288,7 @@ pipeline {
277288

278289
stage('Build-Image') {
279290
steps {
280-
sh "make build version=${mlVersion}-${env.platformString}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
291+
buildDockerImage()
281292
}
282293
}
283294

@@ -289,7 +300,7 @@ pipeline {
289300

290301
stage('Scan') {
291302
steps {
292-
scan()
303+
vulnerabilityScan()
293304
}
294305
}
295306

@@ -307,7 +318,7 @@ pipeline {
307318
expression { return params.DOCKER_TESTS }
308319
}
309320
steps {
310-
sh "make docker-tests test_image=local-dev/marklogic-server-centos:${mlVersion}-${env.platformString}-${env.dockerVersion} version=${mlVersion}-${env.platformString}-${env.dockerVersion} build_branch=${env.BRANCH_NAME}"
321+
dockerTests()
311322
}
312323
}
313324

‎Makefile

Lines changed: 25 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,65 @@
11
version?=internal
22
package?=MarkLogic.rpm
3-
REPONAME=local-dev
4-
docker_registry?=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com
5-
repoDir=marklogic
3+
repo_dir=marklogic
64
docker_build_options=--compress
7-
test_image?=${docker_registry}/${repoDir}/marklogic-server-centos:${version}
85
build_branch?=local
6+
docker_image_type?=ubi
7+
current_image?=${repo_dir}/marklogic-server-${docker_image_type}:${version}
98

109
#***************************************************************************
11-
# build centos docker images
10+
# build docker image
1211
#***************************************************************************
13-
build-centos:
14-
cd src/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-deps-centos:${version}" -f ../dockerFiles/marklogic-deps-centos:base .
15-
cd src/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-server-centos:${version}" --build-arg BASE_IMAGE=${REPONAME}/marklogic-deps-centos:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-centos:base .
16-
17-
#***************************************************************************
18-
# build ubi docker images
19-
#***************************************************************************
20-
build-ubi:
21-
cd src/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-deps-ubi:${version}" -f ../dockerFiles/marklogic-deps-ubi:base .
22-
cd src/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-server-ubi:${version}" --build-arg BASE_IMAGE=${REPONAME}/marklogic-deps-ubi:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-ubi:base .
23-
24-
#***************************************************************************
25-
# build ubi rootless docker images
26-
#***************************************************************************
27-
build-ubi-rootless:
28-
cd src/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-deps-ubi:${version}" -f ../dockerFiles/marklogic-deps-ubi:base .
29-
cd src/; docker build ${docker_build_options} -t "${REPONAME}/marklogic-server-ubi-rootless:${version}" --build-arg BASE_IMAGE=${REPONAME}/marklogic-deps-ubi:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-ubi-rootless:base .
30-
12+
build:
13+
#ubi-rootless uses the same dependencies as ubi image
14+
ifeq ($(docker_image_type),ubi-rootless)
15+
cp dockerFiles/marklogic-deps-ubi\:base dockerFiles/marklogic-deps-ubi-rootless\:base
16+
endif
17+
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-deps-${docker_image_type}:${version}" -f ../dockerFiles/marklogic-deps-${docker_image_type}:base .
18+
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-server-${docker_image_type}:${version}" --build-arg BASE_IMAGE=${repo_dir}/marklogic-deps-${docker_image_type}:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-${docker_image_type}:base .
19+
rm -f dockerFiles/marklogic-deps-ubi-rootless\:base
3120
#***************************************************************************
3221
# strcture test docker images
3322
#***************************************************************************
3423
structure-test:
35-
container-structure-test test --config ./test/structure-test.yaml --image TEST_IMAGE:${test_image} \
24+
container-structure-test test --config ./test/structure-test.yaml --image ${current_image} \
3625
$(if $(Jenkins), --output junit | tee container-structure-test.xml,)
3726

3827
#***************************************************************************
3928
# docker image tests
4029
#***************************************************************************
4130
docker-tests:
4231
cd test; python3 -m venv python_env
43-
cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --variable TEST_IMAGE:${test_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --maxerrorlines 9999 ./docker-tests.robot; deactivate
32+
cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --variable TEST_IMAGE:${current_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --maxerrorlines 9999 ./docker-tests.robot; deactivate
4433
rm -r test/python_env/
4534

4635
#***************************************************************************
4736
# run all tests
4837
#***************************************************************************
4938
.PHONY: test
5039
test: structure-test docker-tests
51-
52-
#***************************************************************************
53-
# push docker images to mlregistry.marklogic.com
54-
#***************************************************************************
55-
push-mlregistry:
56-
docker tag ${REPONAME}/marklogic-server-centos:${version} ${docker_registry}/${repoDir}/marklogic-server-centos:${version}
57-
docker push ${docker_registry}/${repoDir}/marklogic-server-centos:${version}
5840

5941
#***************************************************************************
60-
# run lint checker on Dockerfiles, print linting issues but do not fail the build
42+
# run lint checker on shell scripts and Dockerfiles, print linting issues but do not fail the build
6143
#***************************************************************************
44+
6245
lint:
63-
docker run --rm -v "${PWD}:/mnt" koalaman/shellcheck:stable src/scripts/start-marklogic.sh $(if $(Jenkins), > start-marklogic-lint.txt,)
64-
docker run --rm -v "${PWD}:/mnt" koalaman/shellcheck:stable src/scripts/start-marklogic-rootless.sh $(if $(Jenkins), >> start-marklogic-lint.txt,)
65-
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/marklogic-deps-centos:base $(if $(Jenkins), > marklogic-deps-centos-base-lint.txt,)
66-
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/marklogic-server-centos:base $(if $(Jenkins), > marklogic-server-centos-base-lint.txt,)
67-
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/marklogic-server-ubi:base $(if $(Jenkins), >> marklogic-server-centos-base-lint.txt,)
68-
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/marklogic-server-ubi-rootless:base $(if $(Jenkins), >> marklogic-server-centos-base-lint.txt,)
46+
docker run --rm -v "${PWD}:/mnt" koalaman/shellcheck:stable src/scripts/*.sh $(if $(Jenkins), > start-scripts-lint.txt,)
47+
48+
@for dockerFile in $(shell ls ./dockerFiles/); do\
49+
echo "Lint results for $${dockerFile}" $(if $(Jenkins), >> dockerfile-lint.txt,) ; \
50+
docker run --rm -i -v "${PWD}/hadolint.yaml":/.config/hadolint.yaml ghcr.io/hadolint/hadolint < dockerFiles/$${dockerFile} $(if $(Jenkins), >> dockerfile-lint.txt,);\
51+
echo $(if $(Jenkins), >> dockerfile-lint.txt,) ;\
52+
done
6953

7054
#***************************************************************************
7155
# security scan docker images
7256
#***************************************************************************
7357
scan:
74-
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock anchore/grype:latest TEST_IMAGE:${test_image} $(if $(Jenkins), > scan-server-image.txt,)
58+
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock anchore/grype:latest ${current_image} $(if $(Jenkins), > scan-server-image.txt,)
7559

7660
#***************************************************************************
7761
# remove junk
7862
#***************************************************************************
7963
clean:
8064
rm -f *.log
81-
rm -f *.rpm
65+
rm -f *.rpm

‎dockerFiles/marklogic-deps-centos:base

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ LABEL "com.marklogic.maintainer"="[email protected]"
1313

1414
RUN echo "NETWORKING=yes" > /etc/sysconfig/network \
1515
&& yum -y update \
16-
&& yum -y install gdb.x86_64 glibc.i686 initscripts redhat-lsb-core.x86_64 tzdata xz-0:5.2.2-2.el7_9.x86_64 \
16+
&& yum -y install gdb.x86_64 glibc.i686 libstdc++.i686 libgcc.i686 initscripts redhat-lsb-core.x86_64 tzdata xz-0:5.2.2-2.el7_9.x86_64 \
1717
&& yum clean all
1818

1919
###############################################################

‎src/scripts/start-marklogic-rootless.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ if [[ "${INSTALL_CONVERTERS}" == "true" ]]; then
8080
else
8181
info "INSTALL_CONVERTERS is true, installing converters."
8282
CONVERTERS_PATH="/tmp/converters.rpm"
83-
cd /tmp
83+
cd /tmp || exit
8484
rpm2cpio ${CONVERTERS_PATH} | cpio -ivd ./opt/*
8585
mv /tmp/opt/MarkLogic/Converters /opt/MarkLogic/
8686
rm -rf /tmp/opt && rm -rf ${CONVERTERS_PATH}

0 commit comments

Comments
 (0)