diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index ef56fcf2d0a..d82ddd10dfb 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -8,6 +8,11 @@ function run_test() { local sketchdir=$(dirname $sketch) local sketchname=$(basename $sketchdir) + if [[ -f "$sketchdir/.skip.$platform" ]] || [[ -f "$sketchdir/.skip.$target" ]] || [[ -f "$sketchdir/.skip.$platform.$target" ]]; then + echo "Skipping $sketchname test in $target for $platform" + exit 0 + fi + if [ $options -eq 0 ] && [ -f $sketchdir/cfg.json ]; then len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json` else @@ -33,7 +38,20 @@ function run_test() { report_file="tests/$sketchname/$sketchname$i.xml" fi - pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file + if [ $platform == "qemu" ]; then + extra_args="--embedded-services qemu --qemu-image-path $build_dir/$sketchname.ino.bin" + + if [ $target == "esp32" ] || [ $target == "esp32s3" ]; then + extra_args+=" --qemu-prog-path $QEMU_PATH/qemu-system-xtensa --qemu-cli-args '-nographic -machine $target'" + else + extra_args+=" --qemu-prog-path $QEMU_PATH/qemu-system-riscv32 --qemu-cli-args '-nographic -machine $target'" + fi + else + extra_args="--embedded-services esp,arduino" + fi + + echo "pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args" + pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args result=$? if [ $result -ne 0 ]; then return $result @@ -44,6 +62,7 @@ function run_test() { SCRIPTS_DIR="./.github/scripts" COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count" +platform="hardware" chunk_run=0 options=0 erase=0 @@ -53,6 +72,13 @@ while [ ! -z "$1" ]; do -c ) chunk_run=1 ;; + -q ) + if [ ! -d $QEMU_PATH ]; then + echo "QEMU path $QEMU_PATH does not exist" + exit 1 + fi + platform="qemu" + ;; -o ) options=1 ;; @@ -86,7 +112,9 @@ while [ ! -z "$1" ]; do shift done -source ${SCRIPTS_DIR}/install-arduino-ide.sh +if [ ! $platform == "qemu" ]; then + source ${SCRIPTS_DIR}/install-arduino-ide.sh +fi if [ $chunk_run -eq 0 ]; then run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index bc3afe4193a..b3c59133718 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -1,4 +1,4 @@ -name: Run tests in hardware +name: Run tests on: pull_request: @@ -16,9 +16,7 @@ concurrency: jobs: gen_chunks: - if: | - contains(github.event.pull_request.labels.*.name, 'hil_test') || - (github.event_name == 'schedule' && github.repository == 'espressif/arduino-esp32') + if: github.repository == 'espressif/arduino-esp32' name: Generate Chunks matrix runs-on: ubuntu-latest outputs: @@ -41,7 +39,7 @@ jobs: CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`) echo "chunks=${CHUNKS}" >>$GITHUB_OUTPUT - Build: + build: needs: gen_chunks name: ${{matrix.chip}}-Build#${{matrix.chunks}} runs-on: ubuntu-latest @@ -63,9 +61,78 @@ jobs: ~/.arduino/tests/*/build*.tmp/*.bin ~/.arduino/tests/*/build*.tmp/*.json if-no-files-found: error - Test: - needs: [gen_chunks, Build] - name: ${{matrix.chip}}-Test#${{matrix.chunks}} + + qemu-test: + needs: [gen_chunks, build] + name: ${{matrix.chip}}-QEMU_Test#${{matrix.chunks}} + strategy: + fail-fast: false + matrix: + chip: ['esp32', 'esp32c3'] # Currently only ESP32 and ESP32-C3 are supported by QEMU + chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}} + runs-on: ubuntu-latest + env: + QEMU_INSTALL_PATH: "$HOME" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get QEMU version + uses: pozetroninc/github-action-get-latest-release@v0.7.0 + id: get-qemu-version + with: + token: ${{secrets.GITHUB_TOKEN}} + owner: espressif + repo: qemu + excludes: prerelease, draft + + - name: Cache tools + id: cache-linux + uses: actions/cache@v4 + with: + path: | + ~/qemu + ~/.cache/pip + key: ${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/hil.yml') }} + + - name: Install python dependencies + run: | + pip install -U pip + pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi + + - name: Download QEMU + if: steps.cache-linux.outputs.cache-hit != 'true' + run: | + cd ${{ env.QEMU_INSTALL_PATH }} + underscore_release=$(echo ${{ steps.get-qemu-version.outputs.release }} | sed 's/\-/_/g') + curl -L https://github.com/espressif/qemu/releases/download/${{ steps.get-qemu-version.outputs.release }}/qemu-riscv32-softmmu-${underscore_release}-x86_64-linux-gnu.tar.xz > qemu-riscv32.tar.xz + curl -L https://github.com/espressif/qemu/releases/download/${{ steps.get-qemu-version.outputs.release }}/qemu-xtensa-softmmu-${underscore_release}-x86_64-linux-gnu.tar.xz > qemu-xtensa.tar.xz + tar -xf qemu-riscv32.tar.xz + tar -xf qemu-xtensa.tar.xz + rm qemu-* + echo "QEMU_PATH=${{ env.QEMU_INSTALL_PATH }}/qemu" >> $GITHUB_ENV + + - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts + uses: actions/download-artifact@v4 + with: + name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts + path: ~/.arduino/tests/ + + - name: Run Tests + run: QEMU_PATH="${{env.QEMU_PATH}}" bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -q + + - name: Upload test result artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: qemu_results-${{matrix.chip}}-${{matrix.chunks}} + path: tests/*/*.xml + + hardware-test: + needs: [gen_chunks, build] + name: ${{matrix.chip}}-Hardware_Test#${{matrix.chunks}} + if: | + contains(github.event.pull_request.labels.*.name, 'hil_test') || github.event_name == 'schedule' strategy: fail-fast: false matrix: @@ -100,15 +167,13 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test_results-${{matrix.chip}}-${{matrix.chunks}} + name: hw_results-${{matrix.chip}}-${{matrix.chunks}} path: tests/*/*.xml event_file: name: "Event File" - if: | - contains(github.event.pull_request.labels.*.name, 'hil_test') || - github.event_name == 'schedule' - needs: Test + if: ${{ always() && !failure() && !cancelled() }} + needs: [hardware-test, qemu-test] runs-on: ubuntu-latest steps: - name: Upload diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 34d3564c4a8..026c9d59094 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Unit Test Results on: workflow_run: - workflows: [Run tests in hardware] + workflows: [Run tests] branches-ignore: [master] types: diff --git a/tests/pytest.ini b/tests/pytest.ini index ef7e6d7c5ae..9646d2f554d 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --embedded-services esp,arduino +addopts = --embedded-services esp,arduino,qemu # log related log_cli = True diff --git a/tests/requirements.txt b/tests/requirements.txt index 896699b5752..4a2a09e755c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,3 +3,4 @@ cryptography>=2.1.4 pytest-cov pytest-embedded-serial-esp>=1.3.4 pytest-embedded-arduino>=1.3.4 +pytest-embedded-qemu>=1.3.4