diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index 43ac41e2d4d..bb90373b56c 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -101,7 +101,7 @@ function run_test() { fi fi done - printf "\n" + printf "Test return code: $error\n" return $error } @@ -250,6 +250,7 @@ else exit_code=0 run_test $target $sketch $options $erase || exit_code=$? + echo "Sketch $sketch exit code: $exit_code" if [ $exit_code -ne 0 ]; then error=$exit_code fi diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index a142342748a..17cd3b0a1bc 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -47,11 +47,24 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout Repository - uses: actions/checkout@v4 + - name: Checkout user repository if: ${{ steps.check-build.outputs.enabled == 'true' }} + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github + + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-build.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github - name: Get libs cache uses: actions/cache@v4 diff --git a/.github/workflows/hw.yml b/.github/workflows/hw.yml index 4be0abe6669..376c38a6530 100644 --- a/.github/workflows/hw.yml +++ b/.github/workflows/hw.yml @@ -49,18 +49,32 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout repository + - name: Checkout user repository if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github - - uses: actions/setup-python@v5 - if: steps.check-tests.outputs.enabled == 'true' - with: - cache-dependency-path: tests/requirements.txt - cache: 'pip' - python-version: '3.10.1' + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-tests.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github + + # setup-python currently only works on ubuntu images + # - uses: actions/setup-python@v5 + # if: ${{ steps.check-tests.outputs.enabled == 'true' }} + # with: + # cache-dependency-path: tests/requirements.txt + # cache: 'pip' + # python-version: '3.10.1' - name: Install dependencies if: ${{ steps.check-tests.outputs.enabled == 'true' }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 56393d46202..c4ae017c229 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -28,6 +28,8 @@ jobs: - name: Set up Python 3 uses: actions/setup-python@v5 with: + cache-dependency-path: tools/pre-commit/requirements.txt + cache: 'pip' python-version: "3.x" - name: Get Python version hash @@ -41,11 +43,10 @@ jobs: with: path: | ~/.cache/pre-commit - ~/.cache/pip - key: pre-commit|${{ env.PY_HASH }}|${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml') }} + key: pre-commit-${{ env.PY_HASH }}-${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml', 'tools/pre-commit/requirements.txt') }} - name: Install python dependencies - run: python -m pip install pre-commit docutils + run: python -m pip install -r tools/pre-commit/requirements.txt - name: Get changed files id: changed-files @@ -61,7 +62,6 @@ jobs: with: path: | ~/.cache/pre-commit - ~/.cache/pip key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Push changes using pre-commit-ci-lite diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ca3f2de55cf..05fd1a8a2b8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -48,6 +48,7 @@ jobs: runs-on: ubuntu-latest outputs: build_all: ${{ steps.set-chunks.outputs.build_all }} + build_libraries: ${{ steps.set-chunks.outputs.build_libraries }} build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }} build_idf: ${{ steps.set-chunks.outputs.build_idf }} build_platformio: ${{ steps.set-chunks.outputs.build_platformio }} @@ -103,6 +104,7 @@ jobs: build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }} build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }} + build_libraries=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }} core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }} @@ -136,9 +138,6 @@ jobs: fi echo "" done - else - echo "Unhandled change triggered the build. This should not happen." - exit 1 fi if [[ -n $sketches ]]; then @@ -164,6 +163,7 @@ jobs: chunks+="]" echo "build_all=$build_all" >> $GITHUB_OUTPUT + echo "build_libraries=$build_libraries" >> $GITHUB_OUTPUT echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT echo "build_idf=$build_idf" >> $GITHUB_OUTPUT echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT @@ -182,6 +182,7 @@ jobs: # Ubuntu build-arduino-linux: name: Arduino ${{ matrix.chunk }} on ubuntu-latest + if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_libraries == 'true' }} needs: gen-chunks runs-on: ubuntu-latest strategy: @@ -274,7 +275,10 @@ jobs: build-esp-idf-component: name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }} needs: gen-chunks - if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_idf == 'true' }} + if: | + needs.gen-chunks.outputs.build_all == 'true' || + needs.gen-chunks.outputs.build_libraries == 'true' || + needs.gen-chunks.outputs.build_idf == 'true' runs-on: ubuntu-20.04 strategy: fail-fast: false diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index 629df904126..1b8d4538901 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -45,35 +45,48 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout repository + - name: Checkout user repository + if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 - if: steps.check-tests.outputs.enabled == 'true' with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github + + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-tests.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github - uses: actions/setup-python@v5 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: cache-dependency-path: tests/requirements.txt cache: 'pip' python-version: '3.x' - name: Install Python dependencies - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: | pip install -U pip pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi - name: Install APT dependencies uses: awalsh128/cache-apt-pkgs-action@v1.4.2 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: packages: libpixman-1-0 libnuma1 libglib2.0-0 libslirp0 libsdl2-2.0-0 version: 1.0 - name: Get QEMU version uses: pozetroninc/github-action-get-latest-release@v0.7.0 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} id: get-qemu-version with: token: ${{secrets.GITHUB_TOKEN}} @@ -84,14 +97,14 @@ jobs: - name: Cache QEMU id: cache-qemu uses: actions/cache@v4 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: path: | ~/qemu key: qemu-${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/qemu.yml') }} - name: Download QEMU - if: steps.cache-qemu.outputs.cache-hit != 'true' && steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.cache-qemu.outputs.cache-hit != 'true' && steps.check-tests.outputs.enabled == 'true' }} run: | cd ${{ env.QEMU_INSTALL_PATH }} underscore_release=$(echo ${{ steps.get-qemu-version.outputs.release }} | sed 's/\-/_/g') @@ -103,7 +116,7 @@ jobs: echo "QEMU_PATH=${{ env.QEMU_INSTALL_PATH }}/qemu" >> $GITHUB_ENV - name: Get binaries - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} id: cache-build-binaries uses: actions/cache/restore@v4 with: @@ -115,7 +128,7 @@ jobs: ~/.arduino/tests/**/build*.tmp/*.json - name: Run Tests - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: QEMU_PATH="${{ env.QEMU_INSTALL_PATH }}" bash .github/scripts/tests_run.sh -c -type ${{inputs.type}} -t ${{inputs.chip}} -i 0 -m 1 -Q - name: Upload ${{ inputs.chip }} ${{ inputs.type }} QEMU results as cache diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25243cf9440..d5a8e267c5f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,14 +12,6 @@ on: - '!libraries/**.txt' - '!libraries/**.properties' - 'package/**' - - '.github/workflows/tests.yml' - - '.github/workflows/build_tests.yml' - - '.github/workflows/hw.yml' - - '.github/workflows/wokwi.yml' - - '.github/workflows/qemu.yml' - - '.github/scripts/install-*.sh' - - '.github/scripts/tests_*.sh' - - '.github/scripts/sketch_utils.sh' schedule: - cron: '0 2 * * *' @@ -167,7 +159,7 @@ jobs: clean: name: Clean objects needs: unit-test-results - if: always() && ${{ github.event_name }} == 'pull_request_target' && ${{ github.event.action }} != 'closed' + if: always() permissions: actions: write runs-on: ubuntu-latest @@ -179,6 +171,11 @@ jobs: const ref = '${{ github.event.pull_request.number || github.ref }}'; const key_prefix = 'tests-' + ref + '-'; + if ('${{ github.event_name }}' == 'pull_request_target' && '${{ github.event.action }}' != 'closed') { + console.log('Skipping cache cleanup for open PR'); + return; + } + await github.paginate(github.rest.actions.getActionsCacheList, { owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/wokwi.yml b/.github/workflows/wokwi.yml index ae68daff9e0..8462f47eb12 100644 --- a/.github/workflows/wokwi.yml +++ b/.github/workflows/wokwi.yml @@ -30,16 +30,15 @@ jobs: id: ${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}-${{ inputs.chip }}-${{ inputs.type }} runs-on: ubuntu-latest steps: - # Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually) - # - name: Check if already run - # if: ${{ github.event.pull_request.number != null }} - # id: get-cache-results - # uses: actions/cache/restore@v4 - # with: - # key: tests-${{ env.id }}-results-wokwi - # path: | - # tests/**/*.xml - # tests/**/result_*.json + - name: Check if already run + if: ${{ github.event.pull_request.number != null }} + id: get-cache-results + uses: actions/cache/restore@v4 + with: + key: tests-${{ env.id }}-results-wokwi + path: | + tests/**/*.xml + tests/**/result_*.json - name: Evaluate if tests should be run id: check-tests @@ -54,35 +53,48 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout Repository + - name: Checkout user repository + if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 - if: steps.check-tests.outputs.enabled == 'true' with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github + + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-tests.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github - uses: actions/setup-python@v5 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: cache-dependency-path: tests/requirements.txt cache: 'pip' python-version: '3.x' - name: Install dependencies - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: | pip install -U pip pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi - name: Install Wokwi CLI - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: curl -L https://wokwi.com/ci/install.sh | sh - name: Wokwi CI Server - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: wokwi/wokwi-ci-server-action@v1 - name: Get binaries - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} id: cache-build-binaries uses: actions/cache/restore@v4 with: @@ -94,21 +106,20 @@ jobs: ~/.arduino/tests/**/build*.tmp/*.json - name: Run Tests - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} env: WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }} run: | bash .github/scripts/tests_run.sh -c -type ${{ inputs.type }} -t ${{inputs.chip}} -i 0 -m 1 -W ${{env.WOKWI_TIMEOUT}} - # Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually) - # - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache - # uses: actions/cache/save@v4 - # if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }} - # with: - # key: tests-${{ env.id }}-results-wokwi - # path: | - # tests/**/*.xml - # tests/**/result_*.json + - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache + uses: actions/cache/save@v4 + if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }} + with: + key: tests-${{ env.id }}-results-wokwi + path: | + tests/**/*.xml + tests/**/result_*.json - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as artifacts uses: actions/upload-artifact@v4 diff --git a/tests/validation/gpio/ci.json b/tests/validation/gpio/ci.json index 8e24ae58e5a..13a4b8c1a31 100644 --- a/tests/validation/gpio/ci.json +++ b/tests/validation/gpio/ci.json @@ -1,6 +1,7 @@ { "platforms": { "qemu": false, - "hardware": false + "hardware": false, + "wokwi": false } } diff --git a/tools/pre-commit/requirements.txt b/tools/pre-commit/requirements.txt new file mode 100644 index 00000000000..10db94a1393 --- /dev/null +++ b/tools/pre-commit/requirements.txt @@ -0,0 +1,2 @@ +pre-commit==3.7.1 +docutils==0.21.2