From a47a5818aa6b227b0078523db9420a681f2b18ad Mon Sep 17 00:00:00 2001 From: per1234 Date: Mon, 2 Oct 2023 18:10:37 -0700 Subject: [PATCH 1/5] Use separate attributes for human identifier and runner identifier in build job matrix The "Arduino IDE" GitHub Actions workflow uses a job matrix to make the builds for each host target in parallel. The same steps are used for each job, but some configuration adjustments must be made on a per-target basis. This is done through various attributes in the matrix configuration. Previously the `os` attribute was used for two distinct things: - The machine identifier of the GitHub Actions runner machine of the job. - The differentiator in the human-targeted job name. The attribute name "os" (for "operating system") was misleading because runners are differentiated by more than only the operating system on the machine. The use of a machine identifier as a differentiator in the human-targeted job name was a bad idea because these identifiers would only effectively communicate the nature of a build to humans who are quite knowledgeable about the GitHub Actions workflow syntax. The impact of these poor decisions has not been too severe previously due to there only being a single job for each operating system. However, there is a need for multiple jobs per operating system in order to support multiple host architectures (e.g., macOS x86 and ARM). The solution is to: - Use an appropriate name for the runner identifier attribute. - Use a dedicated attribute for the human friendly job name differentiator. --- .github/workflows/build.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a444f83c0..57266d1ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,24 +61,27 @@ jobs: echo "result=$RESULT" >> $GITHUB_OUTPUT build: - name: build (${{ matrix.config.os }}) + name: build (${{ matrix.config.name }}) needs: run-determination if: needs.run-determination.outputs.result == 'true' strategy: matrix: config: - - os: windows-2019 + - name: Windows # Human identifier for the job. + runs-on: windows-2019 certificate-secret: WINDOWS_SIGNING_CERTIFICATE_PFX # Name of the secret that contains the certificate. certificate-password-secret: WINDOWS_SIGNING_CERTIFICATE_PASSWORD # Name of the secret that contains the certificate password. certificate-extension: pfx # File extension for the certificate. - - os: ubuntu-20.04 - - os: macos-latest + - name: Linux + runs-on: ubuntu-20.04 + - name: macOS x86 + runs-on: macos-latest # APPLE_SIGNING_CERTIFICATE_P12 secret was produced by following the procedure from: # https://www.kencochrane.com/2020/08/01/build-and-sign-golang-binaries-for-macos-with-github-actions/#exporting-the-developer-certificate certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 certificate-password-secret: KEYCHAIN_PASSWORD certificate-extension: p12 - runs-on: ${{ matrix.config.os }} + runs-on: ${{ matrix.config.runs-on }} timeout-minutes: 90 steps: From 9d8f4ceec86058670aa27185a08025054ac64e79 Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 3 Oct 2023 00:32:01 -0700 Subject: [PATCH 2/5] Deduplicate type determination code in build workflow The "Arduino IDE" GitHub Actions workflow is used to generate several distinct types of builds: - Tester builds of commits - Nightly builds - Release builds Different actions must be performed depending on which type of build is being produced. The workflow contains code that uses various criteria to determine the build type. Previously that code was duplicated in multiple places: - The packaging job - The changelog generation job - The nightly build publishing job - The release publishing job This duplication is avoided by moving the code to a dedicated job that makes the build type information available to all subsequent jobs via outputs. --- .github/workflows/build.yml | 63 ++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57266d1ec..f152452ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,10 +60,43 @@ jobs: echo "result=$RESULT" >> $GITHUB_OUTPUT - build: - name: build (${{ matrix.config.name }}) + build-type-determination: needs: run-determination if: needs.run-determination.outputs.result == 'true' + runs-on: ubuntu-latest + outputs: + is-release: ${{ steps.determination.outputs.is-release }} + is-nightly: ${{ steps.determination.outputs.is-nightly }} + permissions: {} + steps: + - name: Determine the type of build + id: determination + run: | + if [[ + "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" + ]]; then + is_release="true" + is_nightly="false" + elif [[ + "${{ github.event_name }}" == "schedule" || + ( + "${{ github.event_name }}" == "workflow_dispatch" && + "${{ github.ref }}" == "refs/heads/main" + ) + ]]; then + is_release="false" + is_nightly="true" + else + is_release="false" + is_nightly="false" + fi + + echo "is-release=$is_release" >> $GITHUB_OUTPUT + echo "is-nightly=$is_nightly" >> $GITHUB_OUTPUT + + build: + name: build (${{ matrix.config.name }}) + needs: build-type-determination strategy: matrix: config: @@ -120,8 +153,8 @@ jobs: AC_TEAM_ID: ${{ secrets.AC_TEAM_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - IS_NIGHTLY: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') }} - IS_RELEASE: ${{ startsWith(github.ref, 'refs/tags/') }} + IS_NIGHTLY: ${{ needs.build-type-determination.outputs.is-nightly }} + IS_RELEASE: ${{ needs.build-type-determination.outputs.is-release }} CAN_SIGN: ${{ secrets[matrix.config.certificate-secret] != '' }} run: | # See: https://www.electron.build/code-signing @@ -190,7 +223,9 @@ jobs: path: ${{ env.JOB_TRANSFER_ARTIFACT }}/${{ matrix.artifact.path }} changelog: - needs: build + needs: + - build-type-determination + - build runs-on: ubuntu-latest outputs: BODY: ${{ steps.changelog.outputs.BODY }} @@ -203,7 +238,7 @@ jobs: - name: Generate Changelog id: changelog env: - IS_RELEASE: ${{ startsWith(github.ref, 'refs/tags/') }} + IS_RELEASE: ${{ needs.build-type-determination.outputs.is-release }} run: | export LATEST_TAG=$(git describe --abbrev=0) export GIT_LOG=$(git log --pretty=" - %s [%h]" $LATEST_TAG..HEAD | sed 's/ *$//g') @@ -229,15 +264,19 @@ jobs: echo "$BODY" > CHANGELOG.txt - name: Upload Changelog [GitHub Actions] - if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') + if: needs.build-type-determination.outputs.is-nightly == 'true' uses: actions/upload-artifact@v3 with: name: ${{ env.JOB_TRANSFER_ARTIFACT }} path: CHANGELOG.txt publish: - needs: changelog - if: github.repository == 'arduino/arduino-ide' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')) + needs: + - build-type-determination + - changelog + if: > + github.repository == 'arduino/arduino-ide' && + needs.build-type-determination.outputs.is-nightly == 'true' runs-on: ubuntu-latest steps: - name: Download [GitHub Actions] @@ -257,8 +296,10 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} release: - needs: changelog - if: startsWith(github.ref, 'refs/tags/') + needs: + - build-type-determination + - changelog + if: needs.build-type-determination.outputs.is-release == 'true' runs-on: ubuntu-latest steps: - name: Download [GitHub Actions] From 284d39f37e997c34a282f8a23291f15ad9138b6e Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 3 Oct 2023 18:44:33 -0700 Subject: [PATCH 3/5] Add native Apple Silicon target to build workflow On every release tag, and manual trigger when the "Include builds on non-free runners" checkbox is checked, make the Arduino IDE build for native Apple Silicon host in addition to the builds that are always generated by the "Arduino IDE" GitHub Actions workflow. Previously, the build workflow only produced a build for x86-64 (AKA "Intel") macOS hosts. Although it is possible to use those builds on Apple Silicon machines via the Rosetta 2 translation software, the performance is significantly inferior to a native build so we must also provide Apple Silicon native builds. Previously the Apple Silicon builds were produced manually. The reason for using that inefficient and error-prone approach instead of the automated continuous deployment system used for other builds was that GitHub did not provide the necessary Apple Silicon runner machines and Arduino was not capable of setting up such self-hosted machines in a manner that would make them feasible for the project maintainers to use. GitHub hosted Apple Silicon runner machines are now available so we can add the target to the build workflow. GitHub gives unlimited use of the basic runner machines for workflow runs in public repositories. However, the macOS ARM architecture is only provided in runner machines which are classified as "larger runner". Use of these runners is charged on a per-minute basis, without any of the free allowances GitHub provides for the normal runners. In order to avoid unnecessary expenditures, native Apple Silicon builds must be generated only when there is compelling reason to do so. Such a build is needed for every release, so the workflow is configured to always generate the builds when triggered by a tag. In addition to releases, Apple Silicon tester builds for pull requests that might have special implications for this target. For this reason, the workflow is configured to allow Apple Silicon builds to be triggered manually by a repository maintainer. The workflow uses a job matrix to run the build for each target on the appropriate runner machine in parallel, using the universally applicable workflow code for all jobs. It uses another job matrix to generate individual workflow artifacts for the tester builds of each target. Previously it was possible to always use the same matrix configurations for all workflow runs. With the addition of the selectively run macOS ARM job, it is now necessary to generate these matrixes on the fly. The electron-updater package used by Arduino IDE's auto-update capability uses a data file (known as the "channel update info file") to check for the availability of updates. A single "channel update info file" is used for the data of the macOS x86 and ARM builds. Since a separate job is used to produce each of those builds, this means the "channel update info file" produced by each of the macOS build jobs must be merged into a single file. --- .github/workflows/build.yml | 308 +++++++++++++++++++++++++++++---- package.json | 1 + scripts/merge-channel-files.js | 73 ++++++++ 3 files changed, 345 insertions(+), 37 deletions(-) create mode 100644 scripts/merge-channel-files.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f152452ad..3f118c2b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,11 +12,17 @@ on: - '.vscode/**' - 'docs/**' - 'scripts/**' + - '!scripts/merge-channel-files.js' - 'static/**' - '*.md' tags: - '[0-9]+.[0-9]+.[0-9]+*' workflow_dispatch: + inputs: + paid-runners: + description: Include builds on non-free runners + type: boolean + default: false pull_request: paths-ignore: - '.github/**' @@ -24,6 +30,7 @@ on: - '.vscode/**' - 'docs/**' - 'scripts/**' + - '!scripts/merge-channel-files.js' - 'static/**' - '*.md' schedule: @@ -32,8 +39,69 @@ on: env: # See vars.GO_VERSION field of https://github.com/arduino/arduino-cli/blob/master/DistTasks.yml GO_VERSION: '1.19' + # See: https://github.com/actions/setup-node/#readme + NODE_VERSION: '18.17' JOB_TRANSFER_ARTIFACT: build-artifacts CHANGELOG_ARTIFACTS: changelog + STAGED_CHANNEL_FILES_ARTIFACT: staged-channel-files + BASE_BUILD_DATA: | + - config: + # Human identifier for the job. + name: Windows + runs-on: windows-2019 + # Name of the secret that contains the certificate. + certificate-secret: WINDOWS_SIGNING_CERTIFICATE_PFX + # Name of the secret that contains the certificate password. + certificate-password-secret: WINDOWS_SIGNING_CERTIFICATE_PASSWORD + # File extension for the certificate. + certificate-extension: pfx + # Quoting on the value is required here to allow the same comparison expression syntax to be used for this + # and the companion needs.select-targets.outputs.merge-channel-files property (output values always have string + # type). + mergeable-channel-file: 'false' + artifacts: + - path: '*Windows_64bit.exe' + name: Windows_X86-64_interactive_installer + - path: '*Windows_64bit.msi' + name: Windows_X86-64_MSI + - path: '*Windows_64bit.zip' + name: Windows_X86-64_zip + - config: + name: Linux + runs-on: ubuntu-20.04 + mergeable-channel-file: 'false' + artifacts: + - path: '*Linux_64bit.zip' + name: Linux_X86-64_zip + - path: '*Linux_64bit.AppImage' + name: Linux_X86-64_app_image + - config: + name: macOS x86 + runs-on: macos-latest + # APPLE_SIGNING_CERTIFICATE_P12 secret was produced by following the procedure from: + # https://www.kencochrane.com/2020/08/01/build-and-sign-golang-binaries-for-macos-with-github-actions/#exporting-the-developer-certificate + certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 + certificate-password-secret: KEYCHAIN_PASSWORD + certificate-extension: p12 + mergeable-channel-file: 'true' + artifacts: + - path: '*macOS_64bit.dmg' + name: macOS_X86-64_dmg + - path: '*macOS_64bit.zip' + name: macOS_X86-64_zip + PAID_RUNNER_BUILD_DATA: | + - config: + name: macOS ARM + runs-on: macos-latest-xlarge + certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 + certificate-password-secret: KEYCHAIN_PASSWORD + certificate-extension: p12 + mergeable-channel-file: 'true' + artifacts: + - path: '*macOS_arm64.dmg' + name: macOS_arm64_dmg + - path: '*macOS_arm64.zip' + name: macOS_arm64_zip jobs: run-determination: @@ -67,6 +135,7 @@ jobs: outputs: is-release: ${{ steps.determination.outputs.is-release }} is-nightly: ${{ steps.determination.outputs.is-nightly }} + channel-name: ${{ steps.determination.outputs.channel-name }} permissions: {} steps: - name: Determine the type of build @@ -77,6 +146,7 @@ jobs: ]]; then is_release="true" is_nightly="false" + channel_name="stable" elif [[ "${{ github.event_name }}" == "schedule" || ( @@ -86,34 +156,101 @@ jobs: ]]; then is_release="false" is_nightly="true" + channel_name="nightly" else is_release="false" is_nightly="false" + channel_name="nightly" fi echo "is-release=$is_release" >> $GITHUB_OUTPUT echo "is-nightly=$is_nightly" >> $GITHUB_OUTPUT + echo "channel-name=$channel_name" >> $GITHUB_OUTPUT + + select-targets: + needs: build-type-determination + runs-on: ubuntu-latest + outputs: + artifact-matrix: ${{ steps.assemble.outputs.artifact-matrix }} + build-matrix: ${{ steps.assemble.outputs.build-matrix }} + merge-channel-files: ${{ steps.assemble.outputs.merge-channel-files }} + permissions: {} + steps: + - name: Assemble target data + id: assemble + run: | + # Only run the builds that incur runner charges on release or select manually triggered runs. + if [[ + "${{ needs.build-type-determination.outputs.is-release }}" == "true" || + "${{ github.event.inputs.paid-runners }}" == "true" + ]]; then + build_matrix="$( + ( + echo "${{ env.BASE_BUILD_DATA }}"; + echo "${{ env.PAID_RUNNER_BUILD_DATA }}" + ) | \ + yq \ + --output-format json \ + '[.[].config]' + )" + + artifact_matrix="$( + ( + echo "${{ env.BASE_BUILD_DATA }}"; + echo "${{ env.PAID_RUNNER_BUILD_DATA }}" + ) | \ + yq \ + --output-format json \ + '[.[].artifacts.[]]' + )" + + # The build matrix produces two macOS jobs (x86 and ARM) so the "channel update info files" + # generated by each must be merged. + merge_channel_files="true" + + else + build_matrix="$( + echo "${{ env.BASE_BUILD_DATA }}" | \ + yq \ + --output-format json \ + '[.[].config]' + )" + + artifact_matrix="$( + echo "${{ env.BASE_BUILD_DATA }}" | \ + yq \ + --output-format json \ + '[.[].artifacts.[]]' + )" + + merge_channel_files="false" + fi + + # Set workflow step outputs. + # See: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + delimiter="$RANDOM" + echo "build-matrix<<$delimiter" >> $GITHUB_OUTPUT + echo "$build_matrix" >> $GITHUB_OUTPUT + echo "$delimiter" >> $GITHUB_OUTPUT + + delimiter="$RANDOM" + echo "artifact-matrix<<$delimiter" >> $GITHUB_OUTPUT + echo "$artifact_matrix" >> $GITHUB_OUTPUT + echo "$delimiter" >> $GITHUB_OUTPUT + + echo "merge-channel-files=$merge_channel_files" >> $GITHUB_OUTPUT build: name: build (${{ matrix.config.name }}) - needs: build-type-determination + needs: + - build-type-determination + - select-targets + env: + # Location of artifacts generated by build. + BUILD_ARTIFACTS_PATH: electron-app/dist/build-artifacts strategy: matrix: - config: - - name: Windows # Human identifier for the job. - runs-on: windows-2019 - certificate-secret: WINDOWS_SIGNING_CERTIFICATE_PFX # Name of the secret that contains the certificate. - certificate-password-secret: WINDOWS_SIGNING_CERTIFICATE_PASSWORD # Name of the secret that contains the certificate password. - certificate-extension: pfx # File extension for the certificate. - - name: Linux - runs-on: ubuntu-20.04 - - name: macOS x86 - runs-on: macos-latest - # APPLE_SIGNING_CERTIFICATE_P12 secret was produced by following the procedure from: - # https://www.kencochrane.com/2020/08/01/build-and-sign-golang-binaries-for-macos-with-github-actions/#exporting-the-developer-certificate - certificate-secret: APPLE_SIGNING_CERTIFICATE_P12 - certificate-password-secret: KEYCHAIN_PASSWORD - certificate-extension: p12 + config: ${{ fromJson(needs.select-targets.outputs.build-matrix) }} runs-on: ${{ matrix.config.runs-on }} timeout-minutes: 90 @@ -121,10 +258,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install Node.js 18.17 + - name: Install Node.js uses: actions/setup-node@v3 with: - node-version: '18.17' + node-version: ${{ env.NODE_VERSION }} registry-url: 'https://registry.npmjs.org' cache: 'yarn' @@ -179,35 +316,131 @@ jobs: yarn --cwd electron-app build yarn --cwd electron-app package + # Both macOS jobs generate a "channel update info file" with same path and name. The second job to complete would + # overwrite the file generated by the first in the workflow artifact. + - name: Stage channel file for merge + if: > + needs.select-targets.outputs.merge-channel-files == 'true' && + matrix.config.mergeable-channel-file == 'true' + run: | + staged_channel_files_path="${{ runner.temp }}/staged-channel-files" + mkdir "$staged_channel_files_path" + mv \ + "${{ env.BUILD_ARTIFACTS_PATH }}/${{ needs.build-type-determination.outputs.channel-name }}-mac.yml" \ + "${staged_channel_files_path}/${{ needs.build-type-determination.outputs.channel-name }}-mac-${{ runner.arch }}.yml" + + # Set workflow environment variable for use in other steps. + # See: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + echo "STAGED_CHANNEL_FILES_PATH=$staged_channel_files_path" >> "$GITHUB_ENV" + + - name: Upload staged-for-merge channel file artifact + uses: actions/upload-artifact@v3 + if: > + needs.select-targets.outputs.merge-channel-files == 'true' && + matrix.config.mergeable-channel-file == 'true' + with: + if-no-files-found: error + name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} + path: ${{ env.STAGED_CHANNEL_FILES_PATH }} + - name: Upload [GitHub Actions] uses: actions/upload-artifact@v3 with: name: ${{ env.JOB_TRANSFER_ARTIFACT }} - path: electron-app/dist/build-artifacts + path: ${{ env.BUILD_ARTIFACTS_PATH }} + + merge-channel-files: + needs: + - build-type-determination + - select-targets + - build + if: needs.select-targets.outputs.merge-channel-files == 'true' + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Set environment variables + run: | + # See: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + echo "CHANNEL_FILES_PATH=${{ runner.temp }}/channel-files" >> "$GITHUB_ENV" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Download staged-for-merge channel files artifact + uses: actions/download-artifact@v3 + with: + name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} + path: ${{ env.CHANNEL_FILES_PATH }} + + - name: Remove no longer needed artifact + uses: geekyeggo/delete-artifact@v2 + with: + name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }} + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: 'https://registry.npmjs.org' + cache: 'yarn' + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install Task + uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + + - name: Install dependencies + run: yarn + + - name: Merge "channel update info files" + run: | + node \ + ./scripts/merge-channel-files.js \ + --channel "${{ needs.build-type-determination.outputs.channel-name }}" \ + --input "${{ env.CHANNEL_FILES_PATH }}" + + - name: Upload merged channel files to job transfer artifact + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: ${{ env.JOB_TRANSFER_ARTIFACT }} + path: ${{ env.CHANNEL_FILES_PATH }} + + # This job serves only as a container for the logic necessary to allow dependent jobs to run if the + # merge-channel-files job was skipped. + merge-channel-files-complete: + needs: + - merge-channel-files + if: > + always() && + ( + needs.merge-channel-files.result == 'skipped' || + needs.merge-channel-files.result == 'success' + ) + runs-on: ubuntu-latest + permissions: {} + steps: + # GitHub Actions requires every job to have >=1 step. + - name: Dummy step + run: '' artifacts: name: ${{ matrix.artifact.name }} artifact - needs: build + needs: + - select-targets + - build if: always() && needs.build.result != 'skipped' runs-on: ubuntu-latest strategy: matrix: - artifact: - - path: '*Linux_64bit.zip' - name: Linux_X86-64_zip - - path: '*Linux_64bit.AppImage' - name: Linux_X86-64_app_image - - path: '*macOS_64bit.dmg' - name: macOS_dmg - - path: '*macOS_64bit.zip' - name: macOS_zip - - path: '*Windows_64bit.exe' - name: Windows_X86-64_interactive_installer - - path: '*Windows_64bit.msi' - name: Windows_X86-64_MSI - - path: '*Windows_64bit.zip' - name: Windows_X86-64_zip + artifact: ${{ fromJson(needs.select-targets.outputs.artifact-matrix) }} steps: - name: Download job transfer artifact @@ -273,6 +506,7 @@ jobs: publish: needs: - build-type-determination + - merge-channel-files-complete - changelog if: > github.repository == 'arduino/arduino-ide' && @@ -298,6 +532,7 @@ jobs: release: needs: - build-type-determination + - merge-channel-files-complete - changelog if: needs.build-type-determination.outputs.is-release == 'true' runs-on: ubuntu-latest @@ -329,8 +564,6 @@ jobs: run: | # See: https://github.com/arduino/arduino-ide/issues/2018 rm "${{ env.JOB_TRANSFER_ARTIFACT }}/stable-linux.yml" - # See: https://github.com/arduino/arduino-ide/issues/408 - rm "${{ env.JOB_TRANSFER_ARTIFACT }}/stable-mac.yml" - name: Publish Release [S3] if: github.repository == 'arduino/arduino-ide' @@ -347,6 +580,7 @@ jobs: # This job must run after all jobs that use the transfer artifact. needs: - build + - merge-channel-files - publish - release - artifacts diff --git a/package.json b/package.json index 2583d816e..70e9c0ac7 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "eslint-plugin-unused-imports": "^2.0.0", "husky": "^6.0.0", "ignore-styles": "^5.0.1", + "js-yaml": "4.1.0", "lerna": "^7.1.4", "lint-staged": "^11.0.0", "node-fetch": "^2.6.1", diff --git a/scripts/merge-channel-files.js b/scripts/merge-channel-files.js new file mode 100644 index 000000000..aa5cc2400 --- /dev/null +++ b/scripts/merge-channel-files.js @@ -0,0 +1,73 @@ +// @ts-check + +const yaml = require('js-yaml'); +const fs = require('fs'); +const path = require('path'); + +if (process.argv.includes('--help') || process.argv.includes('-h')) { + console.log( + `Usage: +merge-channel-files.js [FLAG]... + +Merge the "channel update info files" used by electron-updater. + +Flags: + --channel The name of the update channel. + -h, --help Print help for the script + --input The path of the folder that contains the files to merge. +` + ); + process.exit(0); +} + +const channelFlagIndex = process.argv.indexOf('--channel'); +if (channelFlagIndex < 0) { + console.error('Missing required --channel flag'); + process.exit(1); +} +const channel = process.argv[channelFlagIndex + 1]; +if (!channel) { + console.error('--channel value must be set'); + process.exit(1); +} + +const inputFlagIndex = process.argv.indexOf('--input'); +if (inputFlagIndex < 0) { + console.error('Missing required --input flag'); + process.exit(1); +} +const channelFilesFolder = process.argv[inputFlagIndex + 1]; +if (!channelFilesFolder) { + console.error('--input value must be set'); + process.exit(1); +} + +// Staging file filename suffixes are named according to `runner.arch`. +// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context +const x86ChannelFilePath = path.join( + channelFilesFolder, + channel + '-mac-X64.yml' +); +const arm64ChannelFilePath = path.join( + channelFilesFolder, + channel + '-mac-ARM64.yml' +); + +const x86Data = yaml.load( + fs.readFileSync(x86ChannelFilePath, { encoding: 'utf8' }) +); +const arm64Data = yaml.load( + fs.readFileSync(arm64ChannelFilePath, { encoding: 'utf8' }) +); + +const mergedData = x86Data; +mergedData['files'] = mergedData['files'].concat(arm64Data['files']); + +fs.writeFileSync( + path.join(channelFilesFolder, channel + '-mac.yml'), + yaml.dump(mergedData, { lineWidth: -1 }) +); + +// Clean up by removing staging files. +fs.rmSync(x86ChannelFilePath); +fs.rmSync(arm64ChannelFilePath); From d46869f0287381d2cb38c015737143c6f4c3ccc8 Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 5 Oct 2023 04:35:58 -0700 Subject: [PATCH 4/5] Deduplicate S3 publishing determination code in build workflow The "Arduino IDE" GitHub Actions workflow uploads the nightly and release builds to Amazon S3, from which they are downloaded by the auto-update as well as directly by users via the links on the "Software" page of arduino.cc. The workflow can also be useful in forks. Either by those who want to test contributions staged in their fork prior to submitting a PR to the parent repo, or by those maintaining a hard fork of the project. Even though these forks wouldn't (and couldn't due to lack of access to the encrypted credential secrets only available to the workflow when ran in a trusted context in Arduino's repo)credentials stored in Arduino's repo) use the S3 upload component of the workflow, they may still find it valuable for continuous integration as well as continuous deployment via the tester builds and release builds the workflow also publishes to the GitHub repository it runs in. For this reason, the workflow contains code to determine whether it should attempt the S3 uploads. Previously that code was duplicated in both the nightly and release publishing jobs of the workflow. Since the workflow already contains a job specifically for the purpose of determining the characteristics of the build being performed and making that information available from single source for use throughout the rest of the workflow, it makes sense to also move the S3 upload determination code to that job. --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f118c2b8..c4184a8dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,6 +136,7 @@ jobs: is-release: ${{ steps.determination.outputs.is-release }} is-nightly: ${{ steps.determination.outputs.is-nightly }} channel-name: ${{ steps.determination.outputs.channel-name }} + publish-to-s3: ${{ steps.determination.outputs.publish-to-s3 }} permissions: {} steps: - name: Determine the type of build @@ -166,6 +167,8 @@ jobs: echo "is-release=$is_release" >> $GITHUB_OUTPUT echo "is-nightly=$is_nightly" >> $GITHUB_OUTPUT echo "channel-name=$channel_name" >> $GITHUB_OUTPUT + # Only attempt upload to Amazon S3 if the credentials are available. + echo "publish-to-s3=${{ github.repository == 'arduino/arduino-ide' }}" >> $GITHUB_OUTPUT select-targets: needs: build-type-determination @@ -509,7 +512,7 @@ jobs: - merge-channel-files-complete - changelog if: > - github.repository == 'arduino/arduino-ide' && + needs.build-type-determination.outputs.publish-to-s3 == 'true' && needs.build-type-determination.outputs.is-nightly == 'true' runs-on: ubuntu-latest steps: @@ -566,7 +569,7 @@ jobs: rm "${{ env.JOB_TRANSFER_ARTIFACT }}/stable-linux.yml" - name: Publish Release [S3] - if: github.repository == 'arduino/arduino-ide' + if: needs.build-type-determination.outputs.publish-to-s3 == 'true' uses: docker://plugins/s3 env: PLUGIN_SOURCE: '${{ env.JOB_TRANSFER_ARTIFACT }}/*' From 08cef1de2f305c5138149eef078811d087a4984a Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 5 Oct 2023 04:48:00 -0700 Subject: [PATCH 5/5] Use a generalized criterion for S3 publishing determination in build workflow The "Arduino IDE" GitHub Actions workflow uploads the nightly and release builds to Amazon S3, from which they are downloaded by the auto-update as well as directly by users via the links on the "Software" page of arduino.cc. The workflow can also be useful in forks. Either by those who want to test contributions staged in their fork prior to submitting a PR to the parent repo, or by those maintaining a hard fork of the project. Even though these forks wouldn't (and couldn't due to lack of access to the encrypted credential secrets only available to the workflow when ran in a trusted context in Arduino's repo)credentials stored in Arduino's repo) use the S3 upload component of the workflow, they may still find it valuable for continuous integration as well as continuous deployment via the tester builds and release builds the workflow also publishes to the GitHub repository it runs in. For this reason, the workflow contains code to determine whether it should attempt the S3 uploads. Previously the repository name was used as the criteria in that code. The project specificity of that approach makes the workflow less easily reusable. A more generally applicable criterion is whether the encrypted credential certificate is defined. The new criterion allows the workflow to be used in any repository where the administrator has created an encrypted secret containing their AWS credentials. That might be other projects owned by Arduino, or even 3rd party projects where the owners want to take a similar build publishing approach using their own AWS account. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c4184a8dc..15b1e2906 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -168,7 +168,7 @@ jobs: echo "is-nightly=$is_nightly" >> $GITHUB_OUTPUT echo "channel-name=$channel_name" >> $GITHUB_OUTPUT # Only attempt upload to Amazon S3 if the credentials are available. - echo "publish-to-s3=${{ github.repository == 'arduino/arduino-ide' }}" >> $GITHUB_OUTPUT + echo "publish-to-s3=${{ secrets.AWS_SECRET_ACCESS_KEY != '' }}" >> $GITHUB_OUTPUT select-targets: needs: build-type-determination