diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a8675a013..86436823f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -243,3 +243,89 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.AUTOMATION_GITHUB_TOKEN }} GRADLE_OPTS: '-Dorg.gradle.daemon=false' + + # -------------------------------------------------- + # job: release + # -------------------------------------------------- + release: + name: Release + if: ${{ contains(needs.build.outputs.commit_message,'[release]') }} + runs-on: ubuntu-latest + needs: build + permissions: + contents: write + timeout-minutes: 10 + steps: + # setup steps + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: true + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: 'temurin' + architecture: x64 + + - name: Setup AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-1 + aws-access-key-id: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }} + + - name: Login to Docker hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_ID }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Login to Seqera registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.SEQERA_PUBLIC_CR_URL }} + username: ${{ secrets.SEQERA_PUBLIC_CR_USER }} + password: ${{ secrets.SEQERA_PUBLIC_CR_PASSWORD }} + + # release step + - name: Release + run: bash release/main.sh + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }} + GH_ORG: ${{ vars.PLUGINS_GITHUB_ORG }} + GH_USER: ${{ vars.DEPLOY_GITHUB_USER }} + GH_USER_EMAIL: ${{ vars.DEPLOY_GITHUB_EMAIL }} + GH_TOKEN: ${{ secrets.DEPLOY_GITHUB_TOKEN }} + MAVEN_PUBLISH_URL: ${{ vars.MAVEN_PLUGINS_PUBLISH_URL }} + PLUGINS_INDEX_JSON: ${{ vars.PLUGINS_INDEX_JSON }} + S3_RELEASE_BUCKET: ${{ vars.S3_RELEASE_BUCKET }} + SEQERA_CONTAINER_REGISTRY: ${{ vars.SEQERA_PUBLIC_CR_URL }} + + # upload steps + - name: Upload artifacts (libs) + uses: actions/upload-artifact@v4 + with: + retention-days: 3 + name: libs + path: modules/*/build/libs/ + + - name: Upload artifacts (distribution) + uses: actions/upload-artifact@v4 + with: + retention-days: 3 + name: distribution + path: build/releases/ + + - name: Upload artifacts (plugins) + uses: actions/upload-artifact@v4 + with: + retention-days: 3 + compression-level: 0 + name: plugins + path: | + plugins/build/libs/ + plugins/*/build/libs/ diff --git a/Makefile b/Makefile index 4a12211e00..040877a530 100644 --- a/Makefile +++ b/Makefile @@ -86,10 +86,10 @@ smoke: NXF_SMOKE=1 ./gradlew ${mm}test # -# Upload JAR artifacts to Maven Central +# Generate all the jars required to create a release # -upload: - ./gradlew upload +distribution: + BUILD_PACK=1 ./gradlew buildInfo compile assemble pack javadocJar sourcesJar testFixturesJar # # Create self-contained distribution package @@ -98,38 +98,14 @@ pack: BUILD_PACK=1 ./gradlew pack # -# Upload NF launcher to nextflow.io web site -# -deploy: - BUILD_PACK=1 ./gradlew deploy - -# -# Close artifacts uploaded to Maven central -# -close: - ./gradlew closeAndReleaseRepository - -# -# Upload final package to GitHub +# Initiate the nextflow release process # +.PHONY: release release: - BUILD_PACK=1 ./gradlew release - -# -# Create and upload docker image distribution -# -dockerImage: - BUILD_PACK=1 ./gradlew dockerImage + ./make-release.sh # # Create local docker image # dockerPack: BUILD_PACK=1 ./gradlew publishToMavenLocal dockerPack -Dmaven.repo.local=${PWD}/build/docker/.nextflow/capsule/deps/ - - -upload-plugins: - ./gradlew plugins:upload - -publish-index: - ./gradlew plugins:publishIndex diff --git a/build.gradle b/build.gradle index 4b4ee6e1f2..bfd98929fe 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,6 @@ */ plugins { - id "io.codearte.nexus-staging" version "0.30.0" id 'java' id 'idea' } @@ -274,29 +273,21 @@ task exportClasspath { } } -ext.nexusUsername = project.findProperty('nexusUsername') -ext.nexusPassword = project.findProperty('nexusPassword') -ext.nexusFullName = project.findProperty('nexusFullName') -ext.nexusEmail = project.findProperty('nexusEmail') - // `signing.keyId` property needs to be defined in the `gradle.properties` file ext.enableSignArchives = project.findProperty('signing.keyId') ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-httpfs' ) -configure(coreProjects) { - group = 'io.nextflow' - version = rootProject.file('VERSION').text.trim() -} - /* - * Maven central deployment - * http://central.sonatype.org/pages/gradle.html + * Maven deployment */ configure(coreProjects) { apply plugin: 'maven-publish' apply plugin: 'signing' + group = 'io.nextflow' + version = rootProject.file('VERSION').text.trim() + task javadocJar(type: Jar) { archiveClassifier = 'javadoc' from configurations.groovyDoc @@ -333,9 +324,8 @@ configure(coreProjects) { } developers { developer { - id = nexusUsername - name = nexusFullName - email = nexusEmail + name = 'Paolo Di Tommaso' + email = 'paolo.ditommaso@gmail.com' } } scm { @@ -352,13 +342,12 @@ configure(coreProjects) { repositories { maven { - // change URLs to point to your repos, e.g. http://my.org/repo - def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/" - def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/" - url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl - credentials(PasswordCredentials) { - username nexusUsername - password nexusPassword + url = findProperty('maven_publish_url') ?: System.getenv('MAVEN_PUBLISH_URL') + url = version.endsWith('-SNAPSHOT') ? "$url/snapshots" : "$url/releases" + + credentials(AwsCredentials) { + accessKey = findProperty('aws_access_key_id') ?: System.getenv('AWS_ACCESS_KEY_ID') + secretKey = findProperty('aws_secret_access_key') ?: System.getenv('AWS_SECRET_ACCESS_KEY') } } } @@ -368,14 +357,10 @@ configure(coreProjects) { required { enableSignArchives } sign publishing.publications.mavenJava } - } - -String bytesToHex(byte[] bytes) { - StringBuffer result = new StringBuffer(); - for (byte byt : bytes) result.append(Integer.toString((byt & 0xff) + 0x100, 16).substring(1)); - return result.toString(); +task publishToMaven { + dependsOn coreProjects.publish } task makeDigest { doLast { @@ -383,34 +368,15 @@ task makeDigest { doLast { String str = file('nextflow').text // create sha1 digest = java.security.MessageDigest.getInstance("SHA1").digest(str.getBytes()) - file('nextflow.sha1').text = new BigInteger(1, digest).toString(16) + '\n' + file('nextflow.sha1').text = digest.encodeHex().toString() + '\n' // create sha-256 digest = java.security.MessageDigest.getInstance("SHA-256").digest(str.getBytes()) - file('nextflow.sha256').text = bytesToHex(digest) + '\n' + file('nextflow.sha256').text = digest.encodeHex().toString() + '\n' // create md5 digest = java.security.MessageDigest.getInstance("MD5").digest(str.getBytes()) - file('nextflow.md5').text = bytesToHex(digest) + '\n' + file('nextflow.md5').text = digest.encodeHex().toString() + '\n' }} - -task upload { - dependsOn compile - dependsOn makeDigest - dependsOn coreProjects.publish -} - -/* - * Configure Nextflow staging plugin -- https://github.com/Codearte/gradle-nexus-staging-plugin - * It adds the tasks - * - closeRepository - * - releaseRepository - * - closeAndReleaseRepository - */ -nexusStaging { - packageGroup = 'io.nextflow' - delayBetweenRetriesInMillis = 10_000 -} - if( System.env.BUILD_PACK ) { apply from: 'packing.gradle' } diff --git a/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubRepositoryPublisher.groovy b/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubRepositoryPublisher.groovy index 6f5babbe9d..0d4ed66d22 100644 --- a/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubRepositoryPublisher.groovy +++ b/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubRepositoryPublisher.groovy @@ -61,6 +61,7 @@ class GithubRepositoryPublisher extends DefaultTask { String mergeIndex(List mainIndex, Map> pluginsToPublish) { + boolean modified = false for( Map.Entry> item : pluginsToPublish ) { final pluginId = item.key @@ -69,6 +70,7 @@ class GithubRepositoryPublisher extends DefaultTask { if (!indexEntry) { mainIndex.add(new PluginMeta(id: pluginId, releases: pluginReleases)) + modified = true } else { for (PluginRelease rel : pluginReleases) { @@ -79,11 +81,13 @@ class GithubRepositoryPublisher extends DefaultTask { // if not, add to the index if( !indexRel ) { indexEntry.releases << rel + modified = true } // otherwise, verify the checksum matches else if( indexRel.sha512sum != rel.sha512sum ) { if( overwrite ) { indexEntry.releases[index] = rel + modified = true } else { def msg = "Plugin $pluginId@${rel.version} invalid checksum:\n" @@ -97,11 +101,15 @@ class GithubRepositoryPublisher extends DefaultTask { } } - new GsonBuilder() + if ( modified ) { + return new GsonBuilder() .setPrettyPrinting() .disableHtmlEscaping() .create() .toJson(mainIndex) + } else { + return null + } } List parseMainIndex(GithubClient github, String path) { @@ -188,10 +196,13 @@ class GithubRepositoryPublisher extends DefaultTask { logger.quiet("Merging index") final result = mergeIndex(mainIndex, pluginsToPublish) - // push to github - logger.quiet("Publish merged index to $indexUrl") - - github.pushChange(targetFileName, result.toString() + '\n', "Nextflow plugins update") + if ( result ) { + // push to github + logger.quiet("Publish merged index to $indexUrl") + github.pushChange(targetFileName, result.toString() + '\n', "Nextflow plugins update") + } else { + logger.quiet("No changes to index") + } } } diff --git a/docker/Makefile b/docker/Makefile index e65294f7fe..3c19a9d3e6 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -23,12 +23,6 @@ build-arm: dist/docker/arm64 cp ../nextflow . docker buildx build --platform linux/arm64 --output=type=docker --progress=plain --tag nextflow/nextflow:${version} --build-arg TARGETPLATFORM=linux/arm64 . -release: build - docker push nextflow/nextflow:${version} - # - docker tag nextflow/nextflow:${version} public.cr.seqera.io/nextflow/nextflow:${version} - docker push public.cr.seqera.io/nextflow/nextflow:${version} - #Static builds can now be found at: # # Linux: https://download.docker.com/linux/static diff --git a/make-release.sh b/make-release.sh new file mode 100755 index 0000000000..769e44264d --- /dev/null +++ b/make-release.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -e + +# ----------------------------------------------------------------------------- +# Nextflow release entrypoint +# +# This script starts a Nextflow release, and is executed by the `make release` +# command. It will help guide you through the manual steps required to perform +# a release, and then trigger a release build on the CI system (Github Actions). +# ----------------------------------------------------------------------------- + +cd "$(dirname "$0")" + +# read the nextflow version +read -r NF_VERSION /dev/null 2>&1 \ + && release_exists=true + + # if not exists, create github release, with zip & meta json files + if [[ $release_exists == true ]]; then + echo "Plugin $plugin_name $plugin_version already deployed to github, skipping" + else + gh release create \ + --repo "$plugin_repo" \ + --title "Version $plugin_version" \ + "$plugin_version" \ + "$plugin/build/libs/$plugin_name-$plugin_version.zip" \ + "$plugin/build/libs/$plugin_name-$plugin_version-meta.json" + fi + fi +done + +echo "Done" diff --git a/release/deploy-plugins-to-maven.sh b/release/deploy-plugins-to-maven.sh new file mode 100755 index 0000000000..238093625a --- /dev/null +++ b/release/deploy-plugins-to-maven.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e + +# change to the project root +cd "$(dirname "$0")/.." + +echo " +--------------------------------- +-- Publishing plugins to maven -- +--------------------------------- +" + +# the release process should have already built the jars, so to avoid re-compiling everything +# we can tell gradle to skip all non publish/publication related tasks +./gradlew publishPluginsToMaven \ + $( ./gradlew publishPluginsToMaven --dry-run | grep -iv 'publish\|publication' | awk '/^:/ { print "-x" $1 }') + +echo "Done" diff --git a/release/deploy-to-docker.sh b/release/deploy-to-docker.sh new file mode 100755 index 0000000000..4779582f73 --- /dev/null +++ b/release/deploy-to-docker.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e + +# change to the project root +cd "$(dirname "$0")/.." + +# read the nextflow version +read -r NF_VERSION /dev/null 2>&1 \ + && release_exists=true + +if [[ $release_exists == true ]]; then + echo "Version $NF_VERSION already deployed to S3, skipping" + exit +fi + +# collect files to deploy +files=(build/releases/nextflow-"$NF_VERSION"-*) +if [[ ${#files[@]} -eq 0 ]]; then + echo "ERROR - can't find any files to upload" + exit 1 +fi +files+=( + 'nextflow' + 'nextflow.sha1' + 'nextflow.sha256' + 'nextflow.md5' +) + +# upload them to s3 bucket +for file in "${files[@]}"; do + filename=$(basename "$file") + aws s3 cp "$file" "s3://$S3_RELEASE_BUCKET/$S3_RELEASE_DIR/$filename" \ + --no-progress \ + --storage-class STANDARD \ + --region eu-west-1 \ + --acl public-read +done + +echo "Done" diff --git a/release/main.sh b/release/main.sh new file mode 100644 index 0000000000..2ecee62fe1 --- /dev/null +++ b/release/main.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -e + +# ----------------------------------------------------------------------------- +# Nextflow release script +# +# This is the orchestration script for Nextflow releases. +# +# It is intended to be run by 'headless' CI environments (eg Github Actions) +# to execute a release.You probably don't want to run this script directly. +# +# Instead, use the `make release` command to be guided through the process. +# ----------------------------------------------------------------------------- + +# set defaults for unspecified env vars +export GH_ORG=${GH_ORG:-'nextflow-io'} +export GH_USER=${GH_USER:-'pditommaso'} # TODO - use a service user for releases +export GH_USER_EMAIL=${GH_USER_EMAIL:-'paolo.ditommaso@gmail.com'} + +export MAVEN_PUBLISH_URL=${MAVEN_PUBLISH_URL:-'s3://maven.seqera.io'} +export PLUGINS_INDEX_JSON=${PLUGINS_INDEX_JSON:-'https://github.com/nextflow-io/plugins/main/plugins.json'} +export S3_RELEASE_BUCKET=${S3_RELEASE_BUCKET:-'www2.nextflow.io'} +export SEQERA_CONTAINER_REGISTRY=${SEQERA_CONTAINER_REGISTRY:-'public.cr.seqera.io'} + +# change to the project root +cd "$(dirname "$0")/.." + +# build artifacts +make distribution + +# tag release +./release/tag-release.sh + +# deploy to maven +./release/deploy-to-maven.sh + +# deploy to S3 +./release/deploy-to-s3.sh + +# deploy to docker +./release/deploy-to-docker.sh + +# deploy to github +./release/deploy-to-github.sh + +# deploy plugins +./release/deploy-plugins-to-maven.sh +./release/deploy-plugins-to-github.sh +./release/update-plugins-index.sh + +# finally, publish the new launcher +./release/publish-launcher-script.sh diff --git a/release/publish-launcher-script.sh b/release/publish-launcher-script.sh new file mode 100755 index 0000000000..7ee69ec937 --- /dev/null +++ b/release/publish-launcher-script.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e + +# change to the project root +cd "$(dirname "$0")/.." + +# read the nextflow version +read -r NF_VERSION