Skip to content

Commit 69ecf7f

Browse files
vitalykorolevvitalykorolev
committed
- Remove docker script version from published tag
- Fix scheduled trigger MLE-13162 - Change default image to ubi-rootless - Add hadolint exception DL3006 - Update README with the new Docker tags - Improve email notification
1 parent 15ae005 commit 69ecf7f

13 files changed

+91
-75
lines changed

‎Jenkinsfile

+35-22
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/
1515
JIRA_ID = ''
1616
LINT_OUTPUT = ''
1717
SCAN_OUTPUT = ''
18-
IMAGE_INFO = 0
18+
IMAGE_SIZE = 0
1919

2020
// Define local funtions
2121
void preBuildCheck() {
@@ -104,8 +104,8 @@ void resultNotification(message) {
104104
emailList = params.emailList
105105
}
106106
jira_link = "https://project.marklogic.com/jira/browse/${JIRA_ID}"
107-
email_body = "<b>Jenkins pipeline for</b> ${env.JOB_NAME} <br><b>Build Number: </b>${env.BUILD_NUMBER} <b><br><br>Lint Output: <br></b><pre><code>${LINT_OUTPUT}</code></pre><br><b>Vulnerabilities: </b><pre><code>${SCAN_OUTPUT}</code></pre> <br><b>Image Details: <br></b>${IMAGE_INFO} <br><br><b>Build URL: </b><br>${env.BUILD_URL}"
108-
jira_email_body = "${email_body} <br><br><b>Jira URL: </b><br>${jira_link}"
107+
email_body = "<b>Jenkins pipeline for</b> ${env.JOB_NAME} <br><b>Build Number: </b>${env.BUILD_NUMBER} <b><br><br>Lint Output: <br></b><pre><code>${LINT_OUTPUT}</code></pre><br><b>Vulnerabilities: </b><pre><code>${SCAN_OUTPUT}</code></pre> <br><b>Image Size: <br></b>${IMAGE_SIZE} <br><br><b>Build URL: </b><br><a href='${env.BUILD_URL}'>${env.BUILD_URL}</a>"
108+
jira_email_body = "${email_body} <br><br><b>Jira URL: </b><br><a href='${jira_link}'>${jira_link}</a>"
109109

110110
if (JIRA_ID) {
111111
def comment = [ body: "Jenkins pipeline build result: ${message}" ]
@@ -149,31 +149,33 @@ void copyRPMs() {
149149
fi
150150
"""
151151
script {
152+
// Get the RPM and Converters file names
152153
RPM = sh(returnStdout: true, script: 'cd src;file MarkLogic-*.rpm | cut -d: -f1').trim()
153154
CONVERTERS = sh(returnStdout: true, script: 'cd src;file MarkLogicConverters-*.rpm | cut -d: -f1').trim()
154-
mlVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim()
155+
// Extract MarkLogic version from RPM file name
156+
marklogicVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim()
155157
}
156158
}
157159

158160
void buildDockerImage() {
159-
sh "make build docker_image_type=${dockerImageType} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
160-
currentBuild.displayName = "#${BUILD_NUMBER} ${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
161+
sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
162+
currentBuild.displayName = "#${BUILD_NUMBER}: ${marklogicVersion}-${env.dockerImageType} (${env.dockerVersion})"
161163
}
162164

163165
void structureTests() {
164166
sh """
165167
#install container-structure-test 1.16.0 binary
166168
curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test
167-
make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true
169+
make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true
168170
"""
169171
}
170172

171173
void dockerTests() {
172-
sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} version=${mlVersion}-${env.dockerImageType}-${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${dockerImageType}"
174+
sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType}"
173175
}
174176

175177
void lint() {
176-
IMAGE_INFO = sh(returnStdout: true, script: 'docker images | grep \"marklogic-server-'+"${dockerImageType}"+'\"')
178+
IMAGE_SIZE = sh(returnStdout: true, script: "docker images marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} --format '{{.Repository}}:{{.Tag}}\t{{.Size}}'")
177179

178180
sh """
179181
make lint Jenkins=true
@@ -189,7 +191,7 @@ void lint() {
189191

190192
void vulnerabilityScan() {
191193
sh """
192-
make scan current_image=marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
194+
make scan current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} Jenkins=true
193195
grep \'High\\|Critical\' scan-server-image.txt
194196
"""
195197

@@ -202,15 +204,18 @@ void vulnerabilityScan() {
202204
}
203205

204206
void publishToInternalRegistry() {
205-
currentImage="marklogic/marklogic-server-${dockerImageType}:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}"
206-
mlVerShort=mlVersion.split("\\.")[0]
207+
builtImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}"
208+
publishImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}"
209+
mlVerShort=marklogicVersion.split("\\.")[0]
207210
latestTag="marklogic/marklogic-server-${dockerImageType}:latest-${mlVerShort}"
208211
withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) {
209212
sh """
210213
echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${dockerRegistry}
211-
docker tag ${currentImage} ${dockerRegistry}/${currentImage}
212-
docker tag ${currentImage} ${dockerRegistry}/${latestTag}
213-
docker push ${dockerRegistry}/${currentImage}
214+
docker tag ${builtImage} ${dockerRegistry}/${builtImage}
215+
docker tag ${builtImage} ${dockerRegistry}/${publishImage}
216+
docker tag ${builtImage} ${dockerRegistry}/${latestTag}
217+
docker push ${dockerRegistry}/${builtImage}
218+
docker push ${dockerRegistry}/${publishImage}
214219
docker push ${dockerRegistry}/${latestTag}
215220
"""
216221

@@ -225,8 +230,10 @@ void publishToInternalRegistry() {
225230
]]) {
226231
sh """
227232
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com
228-
docker tag ${currentImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}
229-
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${mlVersion}-${env.dockerImageType}-${env.dockerVersion}
233+
docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
234+
docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
235+
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
236+
docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
230237
"""
231238
}
232239
}
@@ -251,10 +258,15 @@ pipeline {
251258
skipStagesAfterUnstable()
252259
}
253260
triggers {
254-
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=10 dockerImageType=centos
255-
00 04 * * * % marklogicVersion=11 dockerImageType=centos
256-
00 05 * * * % marklogicVersion=12 dockerImageType=centos
257-
00 06 * * * % marklogicVersion=11 dockerImageType=ubi''' : '')
261+
parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 02 * * * % marklogicVersion=11;dockerImageType=centos
262+
00 02 * * * % marklogicVersion=11;dockerImageType=ubi
263+
00 02 * * * % marklogicVersion=11;dockerImageType=ubi-rootless
264+
30 02 * * * % marklogicVersion=10;dockerImageType=centos
265+
30 02 * * * % marklogicVersion=10;dockerImageType=ubi
266+
30 02 * * * % marklogicVersion=10;dockerImageType=ubi-rootless
267+
00 03 * * * % marklogicVersion=12;dockerImageType=centos
268+
00 03 * * * % marklogicVersion=12;dockerImageType=ubi
269+
00 03 * * * % marklogicVersion=12;dockerImageType=ubi-rootless''' : '')
258270
}
259271
environment {
260272
QA_LICENSE_KEY = credentials('QA_LICENSE_KEY')
@@ -263,7 +275,7 @@ pipeline {
263275
parameters {
264276
string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
265277
string(name: 'dockerVersion', defaultValue: '1.1.2', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
266-
choice(name: 'dockerImageType', choices: 'centos\nubi\nubi-rootless', description: 'Platform type for Docker image. Will be made part of the docker image tag')
278+
choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\ncentos', description: 'Platform type for Docker image. Will be made part of the docker image tag')
267279
choice(name: 'marklogicVersion', choices: '11\n12\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')
268280
string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true)
269281
string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true)
@@ -330,6 +342,7 @@ pipeline {
330342
}
331343
steps {
332344
publishToInternalRegistry()
345+
build job: 'MarkLogic-Docker-Kubernetes/docker/docker-nightly-builds-qa', wait: false, parameters: [string(name: 'dockerImageType', value: "${dockerImageType}"), string(name: 'marklogicVersion', value: "${marklogicVersion}")]
333346
}
334347
}
335348
}

‎Makefile

+13-8
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
version?=internal
1+
dockerTag?=internal
22
package?=MarkLogic.rpm
33
repo_dir=marklogic
44
docker_build_options=--compress
55
build_branch?=local
66
docker_image_type?=ubi
7-
current_image?=${repo_dir}/marklogic-server-${docker_image_type}:${version}
7+
current_image?=${repo_dir}/marklogic-server-${docker_image_type}:${dockerTag}
88

99
#***************************************************************************
1010
# build docker image
@@ -14,8 +14,8 @@ build:
1414
ifeq ($(docker_image_type),ubi-rootless)
1515
cp dockerFiles/marklogic-deps-ubi\:base dockerFiles/marklogic-deps-ubi-rootless\:base
1616
endif
17-
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-deps-${docker_image_type}:${version}" -f ../dockerFiles/marklogic-deps-${docker_image_type}:base .
18-
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-server-${docker_image_type}:${version}" --build-arg BASE_IMAGE=${repo_dir}/marklogic-deps-${docker_image_type}:${version} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_VERSION=${version} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-${docker_image_type}:base .
17+
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-deps-${docker_image_type}:${dockerTag}" -f ../dockerFiles/marklogic-deps-${docker_image_type}:base .
18+
cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-server-${docker_image_type}:${dockerTag}" --build-arg BASE_IMAGE=${repo_dir}/marklogic-deps-${docker_image_type}:${dockerTag} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_DOCKER_VERSION=${dockerVersion} --build-arg ML_VERSION=${marklogicVersion} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} -f ../dockerFiles/marklogic-server-${docker_image_type}:base .
1919
rm -f dockerFiles/marklogic-deps-ubi-rootless\:base
2020
#***************************************************************************
2121
# strcture test docker images
@@ -28,7 +28,8 @@ else
2828
@echo type is ${docker_image_type}
2929
sed -i -e 's^DOCKER_PID_PLACEHOLDER^/var/run/MarkLogic.pid^g' ./test/structure-test.yaml
3030
endif
31-
sed -i -e 's^VERSION_PLACEHOLDER^${version}^g' ./test/structure-test.yaml
31+
sed -i -e 's^ML_VERSION_PLACEHOLDER^${marklogicVersion}^g' ./test/structure-test.yaml
32+
sed -i -e 's^ML_DOCKER_VERSION_PLACEHOLDER^${dockerVersion}^g' ./test/structure-test.yaml
3233
sed -i -e 's^BRANCH_PLACEHOLDER^${build_branch}^g' ./test/structure-test.yaml
3334
container-structure-test test --config ./test/structure-test.yaml --image ${current_image} \
3435
$(if $(Jenkins), --output junit | tee container-structure-test.xml,)
@@ -37,9 +38,13 @@ endif
3738
# docker image tests
3839
#***************************************************************************
3940
docker-tests:
40-
cd test; python3 -m venv python_env
41-
cd test; source ./python_env/bin/activate; pip3 install -r requirements.txt; robot -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable MARKLOGIC_VERSION:${version} --variable BUILD_BRANCH:${build_branch} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 ./docker-tests.robot; deactivate
42-
rm -r test/python_env/
41+
cd test; \
42+
python3 -m venv python_env; \
43+
source ./python_env/bin/activate; \
44+
pip3 install -r requirements.txt; \
45+
robot -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable MARKLOGIC_VERSION:${marklogicVersion} --variable BUILD_BRANCH:${build_branch} --variable MARKLOGIC_DOCKER_VERSION:${dockerVersion} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 ./docker-tests.robot; \
46+
deactivate; \
47+
rm -rf python_env
4348

4449
#***************************************************************************
4550
# run all tests

‎README.md

+12-10
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ Note: In order to use the MarkLogic Image you need to request the Developer Lice
3535

3636
# Supported tags
3737

38-
Note: MarkLogic Server Docker images follow a specific tagging format: `{ML release version}-{platform}-{ML Docker release version}`
38+
Note: MarkLogic Server Docker images follow a specific tagging format: `{ML release version}-{platform}`
3939

4040
All Supported Tags: [https://hub.docker.com/r/marklogicdb/marklogic-db/tags](https://hub.docker.com/r/marklogicdb/marklogic-db/tags)
4141

@@ -45,7 +45,7 @@ Docker images are maintained by MarkLogic. Send feedback to the MarkLogic Docker
4545

4646
Supported Docker architectures: x86_64
4747

48-
Base OS: CentOS
48+
Base OS: UBI, UBI-rootless and CentOS
4949

5050
Published image artifact details: https://github.com/marklogic/marklogic-docker, https://hub.docker.com/r/marklogicdb/marklogic-db
5151

@@ -240,7 +240,7 @@ Single node configurations are used primarily on a development machine with a si
240240

241241
Create these files on your host machine: `marklogic-single-node.yaml`, `mldb_admin_username.txt`, `mldb_admin_password.txt`, and `mldb_wallet_password.txt`. Run the example Docker commands from the same directory that the files were created.
242242

243-
**marklogic-centos.yaml**
243+
**marklogic-single-node.yaml**
244244

245245
```
246246
#Docker compose file sample to setup single node cluster
@@ -325,7 +325,7 @@ After the container is initialized, you can access the MarkLogic Query Console o
325325

326326
The following is an example of a three-node MarkLogic server cluster created using Docker compose. Create these files on your host machine: `marklogic-multi-node.yaml`, `mldb_admin_username.txt`, and `mldb_admin_password.txt`. Run example Docker commands from the same directory where the files created.
327327

328-
**marklogic-cluster-centos.yaml**
328+
**marklogic-multi-node.yaml**
329329

330330
```
331331
#Docker compose file sample to setup a three node cluster
@@ -975,31 +975,33 @@ If the process is successful, a message saying the node has left the swarm will
975975

976976
The `marklogic` image tags allow the user to pin their applications to images for a specific release, a specific minor release, a specific major release, or the latest release of MarkLogic Server
977977

978-
## `{ML release version}-{platform}-{ML Docker release version}`
978+
## `{ML release version}-{platform}`
979979

980-
This tag points to the exact version of MarkLogic Server, the base OS, and the supporting scripts version. This allows an application to pin to a very specific version of the image. The image will not be updated without incrementing either the MarkLogic Sever version or the version of the supporting scripts.
980+
This tag points to the exact version of MarkLogic Server and the base OS. This allows an application to pin to a very specific version of the image and base OS (platform).
981981

982-
e.g. `11.0.3-centos-1.0.2` is the MarkLogic Server 11.0.3 release, CentOS, version 1.0.2 of the docker scripts.
982+
Platform can be `centos`, `ubi` (RedHat Universal Base Image) or `ubi-rootless` (RedHat Universal Base Image for rootless containers). When `latest` tag is used, the platform will default to `ubi-rootless`.
983+
984+
e.g. `11.2.0-centos` is the MarkLogic Server 11.2.0 release and CentOS base OS.
983985

984986
## `latest-xx.x`
985987

986-
This tag points to the latest patch release of a specific minor version of MarkLogic Server on CentOS. The image will contain the latest docker supporting scripts and OS patches.
988+
This tag points to the latest patch release of a specific minor version of MarkLogic Server on UBI-rootless.
987989

988990
e.g. `latest-11.0` is the latest patch release of MarkLogic Server 11.0 (11.0.0, 11.0.1, etc.).
989991

990992
For MarkLogic 10, because the numbering scheme was changed, the maintenance release is equivalent to the minor release in MarkLogic 11. Use the `latest-10.0-x` tag to pin to a specific maintenance release of MarkLogic 10.
991993

992994
## `latest-xx`
993995

994-
This tag points to the latest minor and patch release of a specific major version of MarkLogic Server on CentOS. The image will contain the latest supporting scripts and OS patches.
996+
This tag points to the latest minor and patch release of a specific major version of MarkLogic Server on UBI-rootless.
995997

996998
e.g. `latest-11` is the latest patch release of the latest minor release of MarkLogic Server 11 (11.0.0, 11.0.1, 11.1.0, 11.1.1, etc.)
997999

9981000
For MarkLogic 10, because the numbering scheme was changed, the maintenance release is equivalent to the minor release in MarkLogic 11. Use the `latest-10` tag to get the latest patch release of the latest maintenance release MarkLogic 10.
9991001

10001002
## `latest`
10011003

1002-
This tag points to the latest minor, patch, and major release of MarkLogic Server on CentOS. The image will contain the latest supporting scripts and OS patches.
1004+
This tag points to the latest minor, patch, and major release of MarkLogic Server on UBI-rootless.
10031005

10041006
It will pull the latest image and can cross patch, minor or major release numbers (11.0.0, 11.0.1, 11.1.0, 11.1.1, 12.0.0, etc.)
10051007

0 commit comments

Comments
 (0)