From 8ead9d924e75b1aecdb0507e94f1c3374da07dee Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Wed, 13 Dec 2023 09:57:22 -0800 Subject: [PATCH 1/4] Update indexfiles at stage level Signed-off-by: Divya Madala --- .../distribution-build.jenkinsfile | 99 ++++++++++++++----- .../opensearch/distribution-build.jenkinsfile | 92 ++++++++++++----- 2 files changed, 146 insertions(+), 45 deletions(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 02d2f72f06..bb9d87b55b 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -184,7 +184,7 @@ pipeline { String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, "latest"), "builds", "opensearch", "manifest.yml"].join("/") String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_TAR = artifactUrl - env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + env.INDEX_FILE_PATH_X64_TAR = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, tar): ${buildManifestUrl}" echo "artifactUrl (linux, x64, tar): ${artifactUrl}" @@ -225,6 +225,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -310,6 +319,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_RPM = artifactUrl + env.INDEX_FILE_PATH_X64_RPM = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, rpm): ${buildManifestUrl}" echo "artifactUrl (linux, x64, rpm): ${artifactUrl}" @@ -328,6 +338,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -412,6 +431,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_DEB = artifactUrl + env.INDEX_FILE_PATH_X64_DEB = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, deb): ${buildManifestUrl}" echo "artifactUrl (linux, x64, deb): ${artifactUrl}" @@ -421,6 +441,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -513,6 +542,8 @@ pipeline { String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, "latest"), "builds", "opensearch", "manifest.yml"].join("/") String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_TAR = artifactUrl + env.INDEX_FILE_PATH_ARM64_TAR = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + echo "buildManifestUrl (linux, arm64, tar): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, tar): ${artifactUrl}" @@ -552,6 +583,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -640,6 +680,8 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_RPM = artifactUrl + env.INDEX_FILE_PATH_ARM64_RPM = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + echo "buildManifestUrl (linux, arm64, rpm): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, rpm): ${artifactUrl}" @@ -657,6 +699,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -741,6 +792,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_DEB = artifactUrl + env.INDEX_FILE_PATH_ARM64_DEB = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, arm64, deb): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, deb): ${artifactUrl}" @@ -750,6 +802,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -797,7 +858,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl - env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" echo "artifactUrl (windows, x64, zip): ${artifactUrl}" @@ -805,6 +866,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_ZIP + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -821,26 +891,6 @@ pipeline { } } } - stage('update index file') { - agent { - docker { - label AGENT_LINUX_X64 - image dockerAgent.image - args dockerAgent.args - registryUrl 'https://public.ecr.aws/' - alwaysPull true - } - } - steps { - script { - if (params.UPDATE_LATEST_URL) { - uploadIndexFile( - indexFilePath: env.INDEX_FILE_PATH - ) - } - } - } - } stage('docker build') { when { beforeAgent true @@ -959,10 +1009,13 @@ pipeline { } } } +<<<<<<< HEAD def markStageUnstableIfPluginsFailedToBuild() { def stageLogs = getLogsForStage(stageName: "${STAGE_NAME}") if (stageLogs.any{e -> e.contains('Failed plugins are')}) { unstable('Some plugins failed to build. See the ./build.sh step for logs and more details') } -} \ No newline at end of file +} +======= +>>>>>>> 64015bb7 (Update indexfiles at stage level) diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 8832c9d520..ab919de617 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -174,7 +174,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_TAR = artifactUrl - env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + env.INDEX_FILE_PATH_X64_TAR = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, tar): ${buildManifestUrl}" echo "artifactUrl (linux, x64, tar): ${artifactUrl}" @@ -214,6 +214,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -294,6 +303,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_RPM = artifactUrl + env.INDEX_FILE_PATH_X64_RPM = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, rpm): ${buildManifestUrl}" echo "artifactUrl (linux, x64, rpm): ${artifactUrl}" @@ -312,6 +322,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -391,6 +410,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_DEB = artifactUrl + env.INDEX_FILE_PATH_X64_DEB = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, deb): ${buildManifestUrl}" echo "artifactUrl (linux, x64, deb): ${artifactUrl}" @@ -399,6 +419,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -442,6 +471,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_TAR = artifactUrl + env.INDEX_FILE_PATH_ARM64_TAR = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, arm64, tar): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, tar): ${artifactUrl}" @@ -466,6 +496,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -546,6 +585,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_RPM = artifactUrl + env.INDEX_FILE_PATH_ARM64_RPM = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, arm64, rpm): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, rpm): ${artifactUrl}" @@ -564,6 +604,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -643,6 +692,7 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_DEB = artifactUrl + env.INDEX_FILE_PATH_ARM64_DEB = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, arm64, deb): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, deb): ${artifactUrl}" @@ -651,6 +701,15 @@ pipeline { } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -694,13 +753,22 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl - env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" echo "artifactUrl (windows, x64, zip): ${artifactUrl}" } } post { + success { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_ZIP + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -717,26 +785,6 @@ pipeline { } } } - stage('update index file') { - agent { - docker { - label AGENT_LINUX_X64 - image dockerAgent.image - args dockerAgent.args - registryUrl 'https://public.ecr.aws/' - alwaysPull true - } - } - steps { - script { - if (params.UPDATE_LATEST_URL) { - uploadIndexFile( - indexFilePath: env.INDEX_FILE_PATH - ) - } - } - } - } stage('docker build') { when { beforeAgent true From 89fea3996b862ca2d6912f2bf3e8deead8a25eab Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Wed, 3 Jan 2024 18:22:05 -0800 Subject: [PATCH 2/4] Rebase Signed-off-by: Divya Madala --- jenkins/opensearch-dashboards/distribution-build.jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index bb9d87b55b..db124c5126 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -1009,7 +1009,6 @@ pipeline { } } } -<<<<<<< HEAD def markStageUnstableIfPluginsFailedToBuild() { def stageLogs = getLogsForStage(stageName: "${STAGE_NAME}") @@ -1017,5 +1016,3 @@ def markStageUnstableIfPluginsFailedToBuild() { unstable('Some plugins failed to build. See the ./build.sh step for logs and more details') } } -======= ->>>>>>> 64015bb7 (Update indexfiles at stage level) From f564ca71d3d8caa66158d812ad70d2dbd6abed77 Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Mon, 8 Jan 2024 16:53:08 -0800 Subject: [PATCH 3/4] Update indexfile to unstable stages Signed-off-by: Divya Madala --- .../distribution-build.jenkinsfile | 63 +++++++++++++++++++ .../opensearch/distribution-build.jenkinsfile | 63 +++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index db124c5126..00291e4609 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -234,6 +234,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -347,6 +356,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -450,6 +468,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -592,6 +619,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -708,6 +744,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -811,6 +856,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -875,6 +929,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_ZIP + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index ab919de617..c28b21c94a 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -223,6 +223,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -331,6 +340,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -428,6 +446,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -505,6 +532,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -613,6 +649,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_RPM + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -710,6 +755,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_ARM64_DEB + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( @@ -769,6 +823,15 @@ pipeline { } } } + unstable { + script { + if (params.UPDATE_LATEST_URL) { + uploadIndexFile( + indexFilePath: env.INDEX_FILE_PATH_X64_ZIP + ) + } + } + } always { script { lib.jenkins.Messages.new(this).add( From ccc1edc0e47deccbc8a44b4bfac8d036004654d9 Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Tue, 23 Jan 2024 12:02:35 -0800 Subject: [PATCH 4/4] Bump the build lib version to 6.1.0 Signed-off-by: Divya Madala --- jenkins/opensearch-dashboards/distribution-build.jenkinsfile | 2 +- jenkins/opensearch/distribution-build.jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 00291e4609..21fb0640d4 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@5.12.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@6.1.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index c28b21c94a..70f6beabd1 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@5.12.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@6.1.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ]))