ci(refactor): Improvements and compilation of only related files

.github/scripts/on-push.sh

@@ -10,7 +10,8 @@ function build(){
     local chunk_index=$3
     local chunks_cnt=$4
     local build_log=$5
-    shift; shift; shift; shift; shift;
+    local sketches_file=$6
+    shift; shift; shift; shift; shift; shift;
     local sketches=$*
 
     local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
@@ -23,6 +24,9 @@ function build(){
     if [ "$OS_IS_LINUX" == "1" ]; then
         args+=" -p $ARDUINO_ESP32_PATH/libraries"
         args+=" -i $chunk_index -m $chunks_cnt"
+        if [ -n "$sketches_file" ]; then
+          args+=" -f $sketches_file"
+        fi
         if [ $build_log -eq 1 ]; then
             args+=" -l $build_log"
         fi
@@ -50,6 +54,7 @@ fi
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
 BUILD_LOG=$3
+SKETCHES_FILE=$4
 BUILD_PIO=0
 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
     CHUNK_INDEX=0
@@ -94,12 +99,12 @@ if [ "$BUILD_PIO" -eq 0 ]; then
     fi
 
     #build sketches for different targets
-    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+    build "esp32s3" $FQBN_ESP32S3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32s2" $FQBN_ESP32S2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c3" $FQBN_ESP32C3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c6" $FQBN_ESP32C6 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32h2" $FQBN_ESP32H2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32"   $FQBN_ESP32   "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
 
     if [ "$BUILD_LOG" -eq 1 ]; then
         #remove last comma from the last JSON object

.github/scripts/sketch_utils.sh

@@ -259,22 +259,28 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
     unset options
 }
 
-function count_sketches(){ # count_sketches <path> [target]
+function count_sketches(){ # count_sketches <path> [target] [file]
     local path=$1
     local target=$2
+    local file=$3
 
     if [ $# -lt 1 ]; then
       echo "ERROR: Illegal number of parameters"
       echo "USAGE: ${0} count <path> [target]"
     fi
 
     rm -rf sketches.txt
+    touch sketches.txt
     if [ ! -d "$path" ]; then
-        touch sketches.txt
         return 0
     fi
 
-    local sketches=$(find $path -name *.ino | sort)
+    if [ -n "$file" ]; then
+        local sketches=$(cat $file)
+    else
+        local sketches=$(find $path -name *.ino | sort)
+    fi
+
     local sketchnum=0
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
@@ -338,6 +344,10 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             shift
             log_compilation=$1
             ;;
+        -f )
+            shift
+            sketches_file=$1
+            ;;
         * )
             break
             ;;
@@ -347,7 +357,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
 
     local xtra_opts=$*
 
-    if [ -z $chunk_index ] || [ -z $chunk_max ]; then
+    if [ -z "$chunk_index" ] || [ -z "$chunk_max" ]; then
         echo "ERROR: Invalid chunk paramters"
         echo "$USAGE"
         exit 1
@@ -363,8 +373,13 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     fi
 
     set +e
-    count_sketches "$path" "$target"
-    local sketchcount=$?
+    if [ -n "$sketches_file" ]; then
+        count_sketches "$path" "$target" "$sketches_file"
+        local sketchcount=$?
+    else
+        count_sketches "$path" "$target"
+        local sketchcount=$?
+    fi
     set -e
     local sketches=$(cat sketches.txt)
     rm -rf sketches.txt

.github/workflows/publishsizes.yml

@@ -2,7 +2,7 @@ name: Sizes Results
 
 on:
   workflow_run:
-    workflows: [ESP32 Arduino CI]
+    workflows: [Compilation Tests]
     types:
       - completed
 
@@ -63,7 +63,7 @@ jobs:
         uses: juliangruber/read-file-action@v1
         with:
           path: ./artifacts/sizes-report/pr_num.txt
-    
+
       - name: Report results
         uses: P-R-O-C-H-Y/report-size-deltas@sizes_v2
         with:

.github/workflows/push.yml

@@ -1,47 +1,193 @@
-name: ESP32 Arduino CI
+name: Compilation Tests
 
 on:
   workflow_dispatch:
   push:
     branches:
     - master
     - release/*
+  pull_request:
     paths:
       - 'cores/**'
       - 'libraries/**'
+      - '!libraries/**.md'
+      - '!libraries/**.txt'
+      - '!libraries/**.properties'
+      - '!libraries/**.py'
       - 'package/**'
       - 'tools/**.py'
       - 'platform.txt'
       - 'programmers.txt'
+      - 'idf_component.yml'
+      - 'Kconfig.projbuild'
+      - 'package.json'
       - '.github/workflows/push.yml'
       - '.github/scripts/**'
       - '!.github/scripts/find_*'
       - '!.github/scripts/on-release.sh'
       - '!.github/scripts/tests_*'
       - '!.github/scripts/upload_*'
-  pull_request:
 
 concurrency:
   group: build-${{github.event.pull_request.number || github.ref}}
   cancel-in-progress: true
 
-jobs:
+env:
+  MAX_CHUNKS: 15
 
+jobs:
   cmake-check:
     name: Check cmake file
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - run: bash ./.github/scripts/check-cmakelists.sh
 
+  gen-chunks:
+    name: Generate chunks
+    runs-on: ubuntu-latest
+    outputs:
+      build_all: ${{ steps.set-chunks.outputs.build_all }}
+      build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }}
+      build_idf: ${{ steps.set-chunks.outputs.build_idf }}
+      build_platformio: ${{ steps.set-chunks.outputs.build_platformio }}
+      chunk_count: ${{ steps.set-chunks.outputs.chunk_count }}
+      chunks: ${{ steps.set-chunks.outputs.chunks }}
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@v44
+      with:
+        files_yaml: |
+            core:
+              - '.github/**'
+              - '!.github/scripts/install-platformio-esp32.sh'
+              - 'cores/**'
+              - 'package/**'
+              - 'tools/**'
+              - '!tools/platformio-build.py'
+              - 'platform.txt'
+              - 'programmers.txt'
+            libraries:
+              - 'libraries/**/examples/**'
+              - 'libraries/**/src/**'
+            static_sketeches:
+              - 'libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino'
+              - 'libraries/BLE/examples/Server/Server.ino'
+              - 'libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino'
+              - 'libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino'
+              - 'libraries/NetworkClientSecure/src/**'
+              - 'libraries/BLE/src/**'
+              - 'libraries/Insights/src/**'
+            idf:
+              - 'idf_component.yml'
+              - 'Kconfig.projbuild'
+            platformio:
+              - 'package.json'
+              - '.github/scripts/install-platformio-esp32.sh'
+              - 'tools/platformio-build.py'
+
+    - name: Set chunks
+      id: set-chunks
+      env:
+        LIB_FILES: ${{ steps.changed-files.outputs.libraries_all_changed_files }}
+      run: |
+        build_all=false
+        chunks_count=0
+        is_pr=${{ github.event_name == 'pull_request' }}
+
+        build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }}
+        build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }}
+        build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }}
+
+        core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }}
+        lib_changed=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }}
+
+        if [[ $core_changed == 'true' ]] || [[ $is_pr != 'true' ]]; then
+          echo "Core files changed or not a PR. Building all."
+          build_all=true
+          chunks_count=${{ env.MAX_CHUNKS }}
+        elif [[ $lib_changed == 'true' ]]; then
+          echo "Libraries changed. Building only affected sketches."
+          sketches=""
+          for file in $LIB_FILES; do
+              if [[ $file == *.ino ]]; then
+                  # If file ends with .ino, add it to the list of sketches
+                  echo "Sketch found: $file"
+                  sketches+="$file "
+              elif [[ $(basename $(dirname $file)) == "src" ]]; then
+                  # If file is in a src directory, find all sketches in the parent/examples directory
+                  echo "Library src file found: $file"
+                  lib=$(dirname $(dirname $file))
+                  lib_sketches=$(find $lib/examples -name *.ino)
+                  sketches+="$lib_sketches "
+                  echo "Library sketches: $lib_sketches"
+              else
+                  # If file is in a example folder but it is not a sketch, find all sketches in the current directory
+                  echo "File in example folder found: $file"
+                  sketch=$(find $(dirname $file) -name *.ino)
+                  sketches+="$sketch "
+                  echo "Sketch in example folder: $sketch"
+              fi
+              echo ""
+          done
+        else
+          echo "Unhandled change triggered the build. This should not happen."
+          exit 1
+        fi
+
+        if [[ -n $sketches ]]; then
+          # Remove duplicates
+          sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq)
+          for sketch in $sketches; do
+              echo $sketch >> sketches_found.txt
+              chunks_count=$((chunks_count+1))
+          done
+          echo "Number of sketches found: $chunks_count"
+          echo "Sketches: $sketches"
+
+          if [[ $chunks_count -gt ${{ env.MAX_CHUNKS }} ]]; then
+            echo "More sketches than the allowed number of chunks found. Limiting to ${{ env.MAX_CHUNKS }} chunks."
+            chunks_count=${{ env.MAX_CHUNKS }}
+          fi
+        fi
+
+        chunks='["0"'
+        for i in $(seq 1 $(( $chunks_count - 1 )) ); do
+          chunks+=",\"$i\""
+        done
+        chunks+="]"
+
+        echo "build_all=$build_all" >> $GITHUB_OUTPUT
+        echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT
+        echo "build_idf=$build_idf" >> $GITHUB_OUTPUT
+        echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT
+        echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT
+        echo "chunks=$chunks" >> $GITHUB_OUTPUT
+
+    - name: Upload sketches found
+      if: ${{ steps.set-chunks.outputs.build_all == 'false' }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: sketches_found
+        path: sketches_found.txt
+        overwrite: true
+        if-no-files-found: error
+
   # Ubuntu
   build-arduino-linux:
     name: Arduino ${{ matrix.chunk }} on ubuntu-latest
+    needs: gen-chunks
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        chunk: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+        chunk: ${{ fromJson(needs.gen-chunks.outputs.chunks) }}
 
     steps:
     - uses: actions/checkout@v4
@@ -62,8 +208,19 @@ jobs:
           ./tools/riscv32-*
           ./tools/xtensa-*
 
-    - name: Build Sketches
-      run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15 1
+    - name: Build all sketches
+      if: ${{ needs.gen-chunks.outputs.build_all == 'true' }}
+      run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} ${{ env.MAX_CHUNKS }} 1
+
+    - name: Download sketches found
+      if: ${{ needs.gen-chunks.outputs.build_all == 'false' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: sketches_found
+
+    - name: Build selected sketches
+      if: ${{ needs.gen-chunks.outputs.build_all == 'false' }}
+      run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} ${{ needs.gen-chunks.outputs.chunk_count }} 1 sketches_found.txt
 
     #Upload cli compile json as artifact
     - name: Upload cli compile json
@@ -76,6 +233,8 @@ jobs:
   # Windows and MacOS
   build-arduino-win-mac:
     name: Arduino on ${{ matrix.os }}
+    needs: gen-chunks
+    if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_static_sketches == 'true' }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -93,6 +252,11 @@ jobs:
   # PlatformIO on Windows, Ubuntu and Mac
   build-platformio:
     name: PlatformIO on ${{ matrix.os }}
+    needs: gen-chunks
+    if: |
+      needs.gen-chunks.outputs.build_all == 'true' ||
+      needs.gen-chunks.outputs.build_static_sketches == 'true' ||
+      needs.gen-chunks.outputs.build_platformio == 'true'
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -109,6 +273,8 @@ jobs:
 
   build-esp-idf-component:
     name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
+    needs: gen-chunks
+    if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_idf == 'true' }}
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: Tests
+name: Unit Tests
 
 on:
   workflow_dispatch:
@@ -8,6 +8,9 @@ on:
       - 'tests/**'
       - 'cores/**'
       - 'libraries/**'
+      - '!libraries/**.md'
+      - '!libraries/**.txt'
+      - '!libraries/**.properties'
       - 'package/**'
       - '.github/workflows/tests.yml'
       - '.github/workflows/build_tests.yml'