diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a444f83c0..15b1e2906 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,11 +12,17 @@ on: - '.vscode/**' - 'docs/**' - 'scripts/**' + - '!scripts/merge-channel-files.js' - 'static/**' - '*.md' tags: - '[0-9]+.[0-9]+.[0-9]+*' workflow_dispatch: + inputs: + paid-runners: + description: Include builds on non-free runners + type: boolean + default: false pull_request: paths-ignore: - '.github/**' @@ -24,6 +30,7 @@ on: - '.vscode/**' - 'docs/**' - 'scripts/**' + - '!scripts/merge-channel-files.js' - 'static/**' - '*.md' schedule: @@ -32,8 +39,69 @@ on: env: # See vars.GO_VERSION field of https://github.com/arduino/arduino-cli/blob/master/DistTasks.yml GO_VERSION: '1.19' + # See: https://github.com/actions/setup-node/#readme + NODE_VERSION: '18.17' JOB_TRANSFER_ARTIFACT: build-artifacts CHANGELOG_ARTIFACTS: changelog + STAGED_CHANNEL_FILES_ARTIFACT: staged-channel-files + BASE_BUILD_DATA: | + - config: + # Human identifier for the job. + name: Windows + runs-on: windows-2019 + # Name of the secret that contains the certificate. + certificate-secret: WINDOWS_SIGNING_CERTIFICATE_PFX + # Name of the secret that contains the certificate password. + certificate-password-secret: WINDOWS_SIGNING_CERTIFICATE_PASSWORD + # File extension for the certificate. + certificate-extension: pfx + # Quoting on the value is required here to allow the same comparison expression syntax to be used for this + # and the companion needs.select-targets.outputs.merge-channel-files property (output values always have string + # type). + mergeable-channel-file: 'false' + artifacts: + - path: '*Windows_64bit.exe' + name: Windows_X86-64_interactive_installer + - path: '*Windows_64bit.msi' + name: Windows_X86-64_MSI + - path: '*Windows_64bit.zip' + name: Windows_X86-64_zip + - config: + name: Linux + runs-on: ubuntu-20.04 + mergeable-channel-file: 'false' + artifacts: + - path: '*Linux_64bit.zip' + name: Linux_X86-64_zip + - path: '*Linux_64bit.AppImage' + name: Linux_X86-64_app_image + - config: + name: macOS x86 + runs-on: macos-latest + # APPLE_SIGNING_CERTIFICATE_P12 secret was produced by following the procedure from: + # https://www.kencochrane.com/2020/08/01/build-and-sign-golang-binaries-for-macos-with-github-actions/#exporting-the-developer-certificate + certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 + certificate-password-secret: KEYCHAIN_PASSWORD + certificate-extension: p12 + mergeable-channel-file: 'true' + artifacts: + - path: '*macOS_64bit.dmg' + name: macOS_X86-64_dmg + - path: '*macOS_64bit.zip' + name: macOS_X86-64_zip + PAID_RUNNER_BUILD_DATA: | + - config: + name: macOS ARM + runs-on: macos-latest-xlarge + certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 + certificate-password-secret: KEYCHAIN_PASSWORD + certificate-extension: p12 + mergeable-channel-file: 'true' + artifacts: + - path: '*macOS_arm64.dmg' + name: macOS_arm64_dmg + - path: '*macOS_arm64.zip' + name: macOS_arm64_zip jobs: run-determination: @@ -60,35 +128,143 @@ jobs: echo "result=$RESULT" >> $GITHUB_OUTPUT - build: - name: build (${{ matrix.config.os }}) + build-type-determination: needs: run-determination if: needs.run-determination.outputs.result == 'true' + runs-on: ubuntu-latest + outputs: + is-release: ${{ steps.determination.outputs.is-release }} + is-nightly: ${{ steps.determination.outputs.is-nightly }} + channel-name: ${{ steps.determination.outputs.channel-name }} + publish-to-s3: ${{ steps.determination.outputs.publish-to-s3 }} + permissions: {} + steps: + - name: Determine the type of build + id: determination + run: | + if [[ + "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" + ]]; then + is_release="true" + is_nightly="false" + channel_name="stable" + elif [[ + "${{ github.event_name }}" == "schedule" || + ( + "${{ github.event_name }}" == "workflow_dispatch" && + "${{ github.ref }}" == "refs/heads/main" + ) + ]]; then + is_release="false" + is_nightly="true" + channel_name="nightly" + else + is_release="false" + is_nightly="false" + channel_name="nightly" + fi + + echo "is-release=$is_release" >> $GITHUB_OUTPUT + echo "is-nightly=$is_nightly" >> $GITHUB_OUTPUT + echo "channel-name=$channel_name" >> $GITHUB_OUTPUT + # Only attempt upload to Amazon S3 if the credentials are available. + echo "publish-to-s3=${{ secrets.AWS_SECRET_ACCESS_KEY != '' }}" >> $GITHUB_OUTPUT + + select-targets: + needs: build-type-determination + runs-on: ubuntu-latest + outputs: + artifact-matrix: ${{ steps.assemble.outputs.artifact-matrix }} + build-matrix: ${{ steps.assemble.outputs.build-matrix }} + merge-channel-files: ${{ steps.assemble.outputs.merge-channel-files }} + permissions: {} + steps: + - name: Assemble target data + id: assemble + run: | + # Only run the builds that incur runner charges on release or select manually triggered runs. + if [[ + "${{ needs.build-type-determination.outputs.is-release }}" == "true" || + "${{ github.event.inputs.paid-runners }}" == "true" + ]]; then + build_matrix="$( + ( + echo "${{ env.BASE_BUILD_DATA }}"; + echo "${{ env.PAID_RUNNER_BUILD_DATA }}" + ) | \ + yq \ + --output-format json \ + '[.[].config]' + )" + + artifact_matrix="$( + ( + echo "${{ env.BASE_BUILD_DATA }}"; + echo "${{ env.PAID_RUNNER_BUILD_DATA }}" + ) | \ + yq \ + --output-format json \ + '[.[].artifacts.[]]' + )" + + # The build matrix produces two macOS jobs (x86 and ARM) so the "channel update info files" + # generated by each must be merged. + merge_channel_files="true" + + else + build_matrix="$( + echo "${{ env.BASE_BUILD_DATA }}" | \ + yq \ + --output-format json \ + '[.[].config]' + )" + + artifact_matrix="$( + echo "${{ env.BASE_BUILD_DATA }}" | \ + yq \ + --output-format json \ + '[.[].artifacts.[]]' + )" + + merge_channel_files="false" + fi + + # Set workflow step outputs. + # See: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + delimiter="$RANDOM" + echo "build-matrix<<$delimiter" >> $GITHUB_OUTPUT + echo "$build_matrix" >> $GITHUB_OUTPUT + echo "$delimiter" >> $GITHUB_OUTPUT + + delimiter="$RANDOM" + echo "artifact-matrix<<$delimiter" >> $GITHUB_OUTPUT + echo "$artifact_matrix" >> $GITHUB_OUTPUT + echo "$delimiter" >> $GITHUB_OUTPUT + + echo "merge-channel-files=$merge_channel_files" >> $GITHUB_OUTPUT + + build: + name: build (${{ matrix.config.name }}) + needs: + - build-type-determination + - select-targets + env: + # Location of artifacts generated by build. + BUILD_ARTIFACTS_PATH: electron-app/dist/build-artifacts strategy: matrix: - config: - - os: windows-2019 - certificate-secret: WINDOWS_SIGNING_CERTIFICATE_PFX # Name of the secret that contains the certificate. - certificate-password-secret: WINDOWS_SIGNING_CERTIFICATE_PASSWORD # Name of the secret that contains the certificate password. - certificate-extension: pfx # File extension for the certificate. - - os: ubuntu-20.04 - - os: macos-latest - # APPLE_SIGNING_CERTIFICATE_P12 secret was produced by following the procedure from: - # https://www.kencochrane.com/2020/08/01/build-and-sign-golang-binaries-for-macos-with-github-actions/#exporting-the-developer-certificate - certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 - certificate-password-secret: KEYCHAIN_PASSWORD - certificate-extension: p12 - runs-on: ${{ matrix.config.os }} + config: ${{ fromJson(needs.select-targets.outputs.build-matrix) }} + runs-on: ${{ matrix.config.runs-on }} timeout-minutes: 90 steps: - name: Checkout uses: actions/checkout@v4 - - name: Install Node.js 18.17 + - name: Install Node.js uses: actions/setup-node@v3 with: - node-version: '18.17' + node-version: ${{ env.NODE_VERSION }} registry-url: 'https://registry.npmjs.org' cache: 'yarn' @@ -117,8 +293,8 @@ jobs: AC_TEAM_ID: ${{ secrets.AC_TEAM_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - IS_NIGHTLY: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') }} - IS_RELEASE: ${{ startsWith(github.ref, 'refs/tags/') }} + IS_NIGHTLY: ${{ needs.build-type-determination.outputs.is-nightly }} + IS_RELEASE: ${{ needs.build-type-determination.outputs.is-release }} CAN_SIGN: ${{ secrets[matrix.config.certificate-secret] != '' }} run: | # See: https://www.electron.build/code-signing @@ -143,35 +319,131 @@ jobs: yarn --cwd electron-app build yarn --cwd electron-app package + # Both macOS jobs generate a "channel update info file" with same path and name. The second job to complete would + # overwrite the file generated by the first in the workflow artifact. + - name: Stage channel file for merge + if: > + needs.select-targets.outputs.merge-channel-files == 'true' && + matrix.config.mergeable-channel-file == 'true' + run: | + staged_channel_files_path="${{ runner.temp }}/staged-channel-files" + mkdir "$staged_channel_files_path" + mv \ + "${{ env.BUILD_ARTIFACTS_PATH }}/${{ needs.build-type-determination.outputs.channel-name }}-mac.yml" \ + "${staged_channel_files_path}/${{ needs.build-type-determination.outputs.channel-name }}-mac-${{ runner.arch }}.yml" + + # Set workflow environment variable for use in other steps. + # See: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + echo "STAGED_CHANNEL_FILES_PATH=$staged_channel_files_path" >> "$GITHUB_ENV" + + - name: Upload staged-for-merge channel file artifact + uses: actions/upload-artifact@v3 + if: > + needs.select-targets.outputs.merge-channel-files == 'true' && + matrix.config.mergeable-channel-file == 'true' + with: + if-no-files-found: error + name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} + path: ${{ env.STAGED_CHANNEL_FILES_PATH }} + - name: Upload [GitHub Actions] uses: actions/upload-artifact@v3 with: name: ${{ env.JOB_TRANSFER_ARTIFACT }} - path: electron-app/dist/build-artifacts + path: ${{ env.BUILD_ARTIFACTS_PATH }} + + merge-channel-files: + needs: + - build-type-determination + - select-targets + - build + if: needs.select-targets.outputs.merge-channel-files == 'true' + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Set environment variables + run: | + # See: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + echo "CHANNEL_FILES_PATH=${{ runner.temp }}/channel-files" >> "$GITHUB_ENV" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Download staged-for-merge channel files artifact + uses: actions/download-artifact@v3 + with: + name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} + path: ${{ env.CHANNEL_FILES_PATH }} + + - name: Remove no longer needed artifact + uses: geekyeggo/delete-artifact@v2 + with: + name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: 'https://registry.npmjs.org' + cache: 'yarn' + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install Task + uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + + - name: Install dependencies + run: yarn + + - name: Merge "channel update info files" + run: | + node \ + ./scripts/merge-channel-files.js \ + --channel "${{ needs.build-type-determination.outputs.channel-name }}" \ + --input "${{ env.CHANNEL_FILES_PATH }}" + + - name: Upload merged channel files to job transfer artifact + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: ${{ env.JOB_TRANSFER_ARTIFACT }} + path: ${{ env.CHANNEL_FILES_PATH }} + + # This job serves only as a container for the logic necessary to allow dependent jobs to run if the + # merge-channel-files job was skipped. + merge-channel-files-complete: + needs: + - merge-channel-files + if: > + always() && + ( + needs.merge-channel-files.result == 'skipped' || + needs.merge-channel-files.result == 'success' + ) + runs-on: ubuntu-latest + permissions: {} + steps: + # GitHub Actions requires every job to have >=1 step. + - name: Dummy step + run: '' artifacts: name: ${{ matrix.artifact.name }} artifact - needs: build + needs: + - select-targets + - build if: always() && needs.build.result != 'skipped' runs-on: ubuntu-latest strategy: matrix: - artifact: - - path: '*Linux_64bit.zip' - name: Linux_X86-64_zip - - path: '*Linux_64bit.AppImage' - name: Linux_X86-64_app_image - - path: '*macOS_64bit.dmg' - name: macOS_dmg - - path: '*macOS_64bit.zip' - name: macOS_zip - - path: '*Windows_64bit.exe' - name: Windows_X86-64_interactive_installer - - path: '*Windows_64bit.msi' - name: Windows_X86-64_MSI - - path: '*Windows_64bit.zip' - name: Windows_X86-64_zip + artifact: ${{ fromJson(needs.select-targets.outputs.artifact-matrix) }} steps: - name: Download job transfer artifact @@ -187,7 +459,9 @@ jobs: path: ${{ env.JOB_TRANSFER_ARTIFACT }}/${{ matrix.artifact.path }} changelog: - needs: build + needs: + - build-type-determination + - build runs-on: ubuntu-latest outputs: BODY: ${{ steps.changelog.outputs.BODY }} @@ -200,7 +474,7 @@ jobs: - name: Generate Changelog id: changelog env: - IS_RELEASE: ${{ startsWith(github.ref, 'refs/tags/') }} + IS_RELEASE: ${{ needs.build-type-determination.outputs.is-release }} run: | export LATEST_TAG=$(git describe --abbrev=0) export GIT_LOG=$(git log --pretty=" - %s [%h]" $LATEST_TAG..HEAD | sed 's/ *$//g') @@ -226,15 +500,20 @@ jobs: echo "$BODY" > CHANGELOG.txt - name: Upload Changelog [GitHub Actions] - if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') + if: needs.build-type-determination.outputs.is-nightly == 'true' uses: actions/upload-artifact@v3 with: name: ${{ env.JOB_TRANSFER_ARTIFACT }} path: CHANGELOG.txt publish: - needs: changelog - if: github.repository == 'arduino/arduino-ide' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')) + needs: + - build-type-determination + - merge-channel-files-complete + - changelog + if: > + needs.build-type-determination.outputs.publish-to-s3 == 'true' && + needs.build-type-determination.outputs.is-nightly == 'true' runs-on: ubuntu-latest steps: - name: Download [GitHub Actions] @@ -254,8 +533,11 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} release: - needs: changelog - if: startsWith(github.ref, 'refs/tags/') + needs: + - build-type-determination + - merge-channel-files-complete + - changelog + if: needs.build-type-determination.outputs.is-release == 'true' runs-on: ubuntu-latest steps: - name: Download [GitHub Actions] @@ -285,11 +567,9 @@ jobs: run: | # See: https://github.com/arduino/arduino-ide/issues/2018 rm "${{ env.JOB_TRANSFER_ARTIFACT }}/stable-linux.yml" - # See: https://github.com/arduino/arduino-ide/issues/408 - rm "${{ env.JOB_TRANSFER_ARTIFACT }}/stable-mac.yml" - name: Publish Release [S3] - if: github.repository == 'arduino/arduino-ide' + if: needs.build-type-determination.outputs.publish-to-s3 == 'true' uses: docker://plugins/s3 env: PLUGIN_SOURCE: '${{ env.JOB_TRANSFER_ARTIFACT }}/*' @@ -303,6 +583,7 @@ jobs: # This job must run after all jobs that use the transfer artifact. needs: - build + - merge-channel-files - publish - release - artifacts diff --git a/package.json b/package.json index 2583d816e..70e9c0ac7 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "eslint-plugin-unused-imports": "^2.0.0", "husky": "^6.0.0", "ignore-styles": "^5.0.1", + "js-yaml": "4.1.0", "lerna": "^7.1.4", "lint-staged": "^11.0.0", "node-fetch": "^2.6.1", diff --git a/scripts/merge-channel-files.js b/scripts/merge-channel-files.js new file mode 100644 index 000000000..aa5cc2400 --- /dev/null +++ b/scripts/merge-channel-files.js @@ -0,0 +1,73 @@ +// @ts-check + +const yaml = require('js-yaml'); +const fs = require('fs'); +const path = require('path'); + +if (process.argv.includes('--help') || process.argv.includes('-h')) { + console.log( + `Usage: +merge-channel-files.js [FLAG]... + +Merge the "channel update info files" used by electron-updater. + +Flags: + --channel The name of the update channel. + -h, --help Print help for the script + --input The path of the folder that contains the files to merge. +` + ); + process.exit(0); +} + +const channelFlagIndex = process.argv.indexOf('--channel'); +if (channelFlagIndex < 0) { + console.error('Missing required --channel flag'); + process.exit(1); +} +const channel = process.argv[channelFlagIndex + 1]; +if (!channel) { + console.error('--channel value must be set'); + process.exit(1); +} + +const inputFlagIndex = process.argv.indexOf('--input'); +if (inputFlagIndex < 0) { + console.error('Missing required --input flag'); + process.exit(1); +} +const channelFilesFolder = process.argv[inputFlagIndex + 1]; +if (!channelFilesFolder) { + console.error('--input value must be set'); + process.exit(1); +} + +// Staging file filename suffixes are named according to `runner.arch`. +// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context +const x86ChannelFilePath = path.join( + channelFilesFolder, + channel + '-mac-X64.yml' +); +const arm64ChannelFilePath = path.join( + channelFilesFolder, + channel + '-mac-ARM64.yml' +); + +const x86Data = yaml.load( + fs.readFileSync(x86ChannelFilePath, { encoding: 'utf8' }) +); +const arm64Data = yaml.load( + fs.readFileSync(arm64ChannelFilePath, { encoding: 'utf8' }) +); + +const mergedData = x86Data; +mergedData['files'] = mergedData['files'].concat(arm64Data['files']); + +fs.writeFileSync( + path.join(channelFilesFolder, channel + '-mac.yml'), + yaml.dump(mergedData, { lineWidth: -1 }) +); + +// Clean up by removing staging files. +fs.rmSync(x86ChannelFilePath); +fs.rmSync(arm64ChannelFilePath);