change: Add ability to switch between two sets of flasher stubs

Author: dobairoland

.github/workflows/build_esptool.yml

@@ -56,7 +56,7 @@ jobs:
           pip install --user -e .
       - name: Build with PyInstaller
         run: |
-          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.STUBS_DIR }}*.json${{ matrix.SEPARATOR }}${{ env.STUBS_DIR }}" esptool.py
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.STUBS_DIR }}1/*.json${{ matrix.SEPARATOR }}${{ env.STUBS_DIR }}1/" --add-data="${{ env.STUBS_DIR }}2/*.json${{ matrix.SEPARATOR }}${{ env.STUBS_DIR }}2/" esptool.py
           pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.EFUSE_DIR }}*.yaml${{ matrix.SEPARATOR }}${{ env.EFUSE_DIR }}" espefuse.py
           pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico espsecure.py
           pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico esp_rfc2217_server.py

.github/workflows/test_esptool.yml

@@ -46,22 +46,21 @@ jobs:
         pytest -m host_test
         pytest test/test_espsecure_hsm.py
 
-    - name: Build stub (Python 3.7 only)
-      if: matrix.python-version == 3.7
-      run: |
-        export TOOLCHAIN_DIR=$HOME/toolchain
-
-        export ESP8266_BINDIR=$TOOLCHAIN_DIR/xtensa-lx106-elf/bin
-        export ESP32_BINDIR=$TOOLCHAIN_DIR/xtensa-esp32-elf/bin
-        export ESP32S2_BINDIR=$TOOLCHAIN_DIR/xtensa-esp32s2-elf/bin
-        export ESP32S3_BINDIR=$TOOLCHAIN_DIR/xtensa-esp32s3-elf/bin
-        export ESP32C3_BINDIR=$TOOLCHAIN_DIR/riscv32-esp-elf/bin
+  check_stubs:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@master
 
-        export PATH=$PATH:$ESP8266_BINDIR:$ESP32_BINDIR:$ESP32S2_BINDIR:$ESP32S3_BINDIR:$ESP32C3_BINDIR
+        - name: Set up Python 3.12
+          uses: actions/setup-python@master
+          with:
+            python-version: 3.12
 
-        ./ci/setup_ci_build_env.sh
-        make -C flasher_stub V=1
-        cd flasher_stub && python ./compare_stubs.py
+        - name: Check if flasher stubs are up-to-date
+          run: |
+            ./ci/download_flasher_stubs.py
+            git diff --exit-code
 
   lint_esptool:
       runs-on: ubuntu-latest

.gitlab-ci.yml

@@ -34,19 +34,6 @@ cache:
   paths:
     - "$CI_PROJECT_DIR/.cache/pip"
 
-.use_esp_flasher_stub: &use_esp_flasher_stub |
-  if [[ "$CI_USE_ESP_FLASHER_STUB" = "1" ]]
-  then
-    apt-get update && apt-get install -y jq
-    wget https://api.github.com/repos/esp-rs/esp-flasher-stub/releases/latest -O /tmp/esp_flasher_stub.json
-    echo "esp-flasher-stub version:"; cat /tmp/esp_flasher_stub.json | jq -r .tag_name
-    mkdir /tmp/assets
-    cat /tmp/esp_flasher_stub.json | jq -r .assets[].browser_download_url | while read -r url; do wget $url -P /tmp/assets; done
-    for f in /tmp/assets/*.json; do fname=$(basename $f); cp $f esptool/targets/stub_flasher/stub_flasher_${fname#esp}; done
-    echo "These are the changes in the repository:"
-    git diff --stat
-  fi
-
 .test_template: &test_template
   stage: test
   image: python:3.7-bullseye
@@ -73,7 +60,7 @@ version_check:
 .host_tests_template: &host_tests_template
   <<: *test_template
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   artifacts:
     when: always
     paths:
@@ -170,7 +157,7 @@ run_pre_commit_hooks:
   stage: test
   image: python:3.7-bullseye
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   tags:
     - host_test
   script:
@@ -192,7 +179,7 @@ run_pre_commit_hooks:
 check_install_coverage:
   <<: *test_template
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   artifacts:
     when: always
     paths:
@@ -210,7 +197,7 @@ check_install_coverage:
 check_install:
   <<: *test_template
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   before_script:
     - pip install .
   script:
@@ -220,7 +207,7 @@ check_install:
 check_install_editable:
   <<: *test_template
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   before_script:
     - pip install -e .
   script:
@@ -230,7 +217,7 @@ check_install_editable:
 check_install_system:
   <<: *test_template
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   before_script:
     - pip install --user .
   script:
@@ -245,41 +232,19 @@ check_install_system:
 check_install_venv:
   <<: *test_template
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   before_script:
     - python -m venv test_env
     - source test_env/bin/activate
     - python -m pip install .
   script:
     - *run_esptool
 
-# Check the stub can build and that a stub built in CI has the same contents
-# as the one embedded in esptool
 check_stub_build:
   <<: *test_template
-  rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
-  artifacts:
-    when: always
-    paths:
-      - flasher_stub/build/
-      - "**/.coverage*"
-      - ".coverage*"
-    expire_in: 1 week
-  variables:
-    TOOLCHAIN_DIR: "/root/.espressif/dist"
-
-    ESP8266_BINDIR: "${TOOLCHAIN_DIR}/xtensa-lx106-elf/bin"
-    ESP32_BINDIR: "${TOOLCHAIN_DIR}/xtensa-esp32-elf/bin"
-    ESP32S2_BINDIR: "${TOOLCHAIN_DIR}/xtensa-esp32s2-elf/bin"
-    ESP32S3_BINDIR: "${TOOLCHAIN_DIR}/xtensa-esp32s3-elf/bin"
-    ESP32C3_BINDIR: "${TOOLCHAIN_DIR}/riscv32-esp-elf/bin"
-
-    TOOLCHAIN_PATHS: "${ESP8266_BINDIR}:${ESP32_BINDIR}:${ESP32S2_BINDIR}:${ESP32S3_BINDIR}:${ESP32C3_BINDIR}"
   script:
-    - ./ci/setup_ci_build_env.sh
-    - make -C flasher_stub V=1 PATH="${TOOLCHAIN_PATHS}:${PATH}"
-    - cd flasher_stub && coverage run --parallel-mode ./compare_stubs.py
+    - ./ci/download_flasher_stubs.py
+    - git diff --exit-code
 
 .target_esptool_test:
   stage: test
@@ -292,7 +257,6 @@ check_stub_build:
     - pip install -e .[dev] --prefer-binary
     # libffi (needed for espsecure) version keeps changing in python docker images. Add a symlink to the installed version on Raspberry Pi
     - if [ $(uname -m) = "armv7l" ]; then ln -sfn /usr/lib/arm-linux-gnueabihf/libffi.so.7.1.0 /usr/lib/arm-linux-gnueabihf/libffi.so.6; fi
-    - *use_esp_flasher_stub
   artifacts:
     reports:
       junit: test/report.xml
@@ -460,7 +424,7 @@ target_esp32p4:
     PYTHONPATH: "$PYTHONPATH:${CI_PROJECT_DIR}/test"
     COVERAGE_PROCESS_START: "${CI_PROJECT_DIR}/test/.covconf"
   rules:
-    - if: $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   before_script:
     - pip install -e .[dev] --prefer-binary
   artifacts:
@@ -526,7 +490,7 @@ build_docs:
   tags:
     - build_docs
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_USE_ESP_FLASHER_STUB == "1"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
       when: never
     - changes:
       - "docs/**/*"
@@ -561,7 +525,7 @@ deploy_docs_preview:
   extends:
     - .deploy_docs_template
   rules:
-    - if: $CI_COMMIT_REF_NAME == "master" || ($CI_PIPELINE_SOURCE == "schedule" && $CI_USE_ESP_FLASHER_STUB == "1")
+    - if: $CI_COMMIT_REF_NAME == "master" || $CI_PIPELINE_SOURCE == "schedule"
       when: never
     - changes:
       - "docs/**/*"
@@ -579,7 +543,7 @@ deploy_docs_production:
   extends:
     - .deploy_docs_template
   rules:
-    - if: $CI_COMMIT_REF_NAME == "master" && $CI_USE_ESP_FLASHER_STUB != "1"
+    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE != "schedule"
       changes:
       - "docs/**/*"
       - "CONTRIBUTING.rst"

MANIFEST.in

@@ -1,11 +1,11 @@
 include README.md
 include LICENSE
-include esptool/targets/stub_flasher/*.json
+include esptool/targets/stub_flasher/1/*
+include esptool/targets/stub_flasher/2/*
 include espefuse/efuse_defs/*.yaml
 # sdist includes test/test*.py by default, but esptool.py tests
 # are so far only intended to run from the git repo itself
 prune test
-prune flasher_stub
 prune .github
 prune docs
 exclude .git*

ci/download_flasher_stubs.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import glob
+import os
+import urllib.request
+
+STUBS = (
+    {
+        "STUB_SET_VERSION": "1",
+        "DOWNLOAD_URL": "https://github.com/espressif/esptool-legacy-flasher-stub/releases/download",
+        "TAG_URL": "https://github.com/espressif/esptool-legacy-flasher-stub/releases/tag",
+        "VERSION": "v1.2.0",
+        "FILE_LIST": (
+            "esp32",
+            "esp32c2",
+            "esp32c3",
+            "esp32c5",
+            "esp32c5beta3",
+            "esp32c6",
+            "esp32c6beta",
+            "esp32h2",
+            "esp32h2beta1",
+            "esp32h2beta2",
+            "esp32p4",
+            "esp32s2",
+            "esp32s3",
+            "esp32s3beta2",
+            "esp8266",
+        ),
+        "LICENSE": "released as Free Software under GNU General Public License Version 2 or later",
+    },
+    {
+        "STUB_SET_VERSION": "2",
+        "DOWNLOAD_URL": "https://github.com/esp-rs/esp-flasher-stub/releases/download",
+        "TAG_URL": "https://github.com/esp-rs/esp-flasher-stub/releases/tag",
+        "VERSION": "v0.3.0",
+        "FILE_LIST": (
+            "esp32",
+            "esp32c2",
+            "esp32c3",
+            "esp32c6",
+            "esp32h2",
+            "esp32s2",
+            "esp32s3",
+        ),
+        "LICENSE": "dual licensed under the Apache License Version 2.0 or the MIT license",
+    },
+)
+
+DESTINATION_DIR = "esptool/targets/stub_flasher"
+
+README_TEMPLATE = """# Licensing
+
+The binaries in JSON format distributed in this directory are {LICENSE}. They were released at {URL} from where the sources can be obtained.
+"""
+
+
+def main():
+    for stub_set in STUBS:
+        dest_sub_dir = os.path.join(DESTINATION_DIR, stub_set["STUB_SET_VERSION"])
+
+        """ The directory is cleaned up so we would detect if a stub was just committed into the repository but the
+        name was not added into the FILE_LIST of STUBS. This would be an unwanted state because the checker would not
+        detect any changes in that stub."""
+        for old_file in glob.glob(os.path.join(dest_sub_dir, "*.json")):
+            print(f"Removing old file {old_file}")
+            os.remove(old_file)
+
+        for file_name in stub_set["FILE_LIST"]:
+            file = ".".join((file_name, "json"))
+            url = "/".join((stub_set["DOWNLOAD_URL"], stub_set["VERSION"], file))
+            dest = os.path.join(dest_sub_dir, file)
+            print(f"Downloading {url} to {dest}")
+            urllib.request.urlretrieve(url, dest)
+
+        with open(os.path.join(dest_sub_dir, "README.md"), "w") as f:
+            print(f"Writing README to {f.name}")
+            f.write(
+                README_TEMPLATE.format(
+                    LICENSE=stub_set["LICENSE"],
+                    URL="/".join((stub_set["TAG_URL"], stub_set["VERSION"])),
+                )
+            )
+
+
+if __name__ == "__main__":
+    main()

ci/setup_ci_build_env.sh

@@ -66,7 +66,7 @@
     write_mem,
 )
 from esptool.config import load_config_file
-from esptool.loader import DEFAULT_CONNECT_ATTEMPTS, ESPLoader, list_ports
+from esptool.loader import DEFAULT_CONNECT_ATTEMPTS, StubFlasher, ESPLoader, list_ports
 from esptool.targets import CHIP_DEFS, CHIP_LIST, ESP32ROM
 from esptool.util import (
     FatalError,
@@ -145,6 +145,15 @@ def main(argv=None, esp=None):
         action="store_true",
     )
 
+    # --stub-version can be set with --no-stub so the tests wouldn't fail if this option is implied globally
+    parser.add_argument(
+        "--stub-version",
+        default=os.environ.get("ESPTOOL_STUB_VERSION", StubFlasher.STUB_SUBDIRS[0]),
+        choices=StubFlasher.STUB_SUBDIRS,
+        # not a public option and is not subject to the semantic versioning policy
+        help=argparse.SUPPRESS,
+    )
+
     parser.add_argument(
         "--trace",
         "-t",
@@ -679,6 +688,8 @@ def add_spi_flash_subparsers(
     print("esptool.py v%s" % __version__)
     load_config_file(verbose=True)
 
+    StubFlasher.set_preferred_stub_subdir(args.stub_version)
+
     # operation function can take 1 arg (args), 2 args (esp, arg)
     # or be a member function of the ESPLoader class.