add support for nightly boards manager packages

.travis.yml

@@ -2,6 +2,8 @@ sudo: false
 language: bash
 os:
   - linux
+env:
+  - BUILD_TYPE=build
 
 addons:
   apt:
@@ -11,40 +13,7 @@ addons:
       - g++-4.8
 
 script:
-  - set -e
-  - export CXX="g++-4.8" CC="gcc-4.8" GCOV="gcov-4.8"
-  - echo -e "travis_fold:start:host_tests"
-  - pushd $TRAVIS_BUILD_DIR/tests/host
-  - make
-  - make clean-objects
-  - echo -e "travis_fold:end:host_tests"
-  - echo -e "travis_fold:start:sketch_test_env_prepare"
-  - popd
-  - wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
-  - tar xf arduino.tar.xz
-  - mv arduino-nightly $HOME/arduino_ide
-  - cd $HOME/arduino_ide/hardware
-  - mkdir esp8266com
-  - cd esp8266com
-  - ln -s $TRAVIS_BUILD_DIR esp8266
-  - cd esp8266/tools
-  - python get.py
-  - export PATH="$HOME/arduino_ide:$TRAVIS_BUILD_DIR/tools/xtensa-lx106-elf/bin:$PATH"
-  - which arduino
-  - cd $TRAVIS_BUILD_DIR
-  - source tests/common.sh
-  - install_libraries
-  - echo -e "travis_fold:end:sketch_test_env_prepare"
-  - echo -e "travis_fold:start:sketch_test"
-  - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries"
-  - echo -e "travis_fold:end:sketch_test"
-  - echo -e "travis_fold:start:size_report"
-  - cat size.log
-  - echo -e "travis_fold:end:size_report"
-
-after_success:
-    - pushd $TRAVIS_BUILD_DIR/tests/host
-    - bash <(curl -s https://codecov.io/bash) -X gcov
+  - $TRAVIS_BUILD_DIR/tests/common.sh
 
 notifications:
   email:

README.md

@@ -38,10 +38,10 @@ Boards manager link: `http://arduino.esp8266.com/stable/package_esp8266com_index
 
 Documentation: [http://esp8266.github.io/Arduino/versions/2.3.0/](http://esp8266.github.io/Arduino/versions/2.3.0/)
 
-##### Staging version ![](http://arduino.esp8266.com/staging/badge.svg)
-Boards manager link: `http://arduino.esp8266.com/staging/package_esp8266com_index.json`
+##### Nightly version ![](http://arduino.esp8266.com/nightly/badge.svg)
+Boards manager link: `http://arduino.esp8266.com/nightly/package_esp8266com_index.json`
 
-Documentation: [http://esp8266.github.io/Arduino/versions/2.3.0-rc2/](http://esp8266.github.io/Arduino/versions/2.3.0-rc2/)
+No online documentation available for nightly version yet — work in progress.
 
 ### Using git version
 [![Linux build status](https://travis-ci.org/esp8266/Arduino.svg)](https://travis-ci.org/esp8266/Arduino) [![codecov.io](https://codecov.io/github/esp8266/Arduino/coverage.svg?branch=master)](https://codecov.io/github/esp8266/Arduino?branch=master)

package/build_boards_manager_package.sh

@@ -1,8 +1,21 @@
 #!/bin/bash
 #
 
+# Extract next version from platform.txt
+next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt`
+
 # Figure out how will the package be called
-ver=`git describe --tags --always`
+ver=`git describe --exact-match`
+if [ $? -ne 0 ]; then
+    # not tagged version; generate nightly package
+    date_str=`date +"%Y%m%d"`
+    is_nightly=1
+    plain_ver="${next}-nightly"
+    ver="${plain_ver}+${date_str}"
+else
+    plain_ver=$ver
+fi
+
 package_name=esp8266-$ver
 echo "Version: $ver"
 echo "Package name: $package_name"
@@ -16,14 +29,14 @@ fi
 echo "Remote: $REMOTE_URL"
 
 if [ -z "$PKG_URL" ]; then
-    PKG_URL="$REMOTE_URL/versions/$ver/$package_name.zip"
+    if [ -z "$PKG_URL_PREFIX" ]; then
+        PKG_URL_PREFIX="$REMOTE_URL/versions/$ver"
+    fi
+    PKG_URL="$PKG_URL_PREFIX/$package_name.zip"
 fi
 echo "Package: $PKG_URL"
-
-if [ -z "$DOC_URL" ]; then
-    DOC_URL="$REMOTE_URL/versions/$ver/doc/reference.html"
-fi
 echo "Docs: $DOC_URL"
+
 pushd ..
 # Create directory for the package
 outdir=package/versions/$ver/$package_name
@@ -37,6 +50,7 @@ cat << EOF > exclude.txt
 .gitignore
 .travis.yml
 package
+doc
 EOF
 # Also include all files which are ignored by git
 git ls-files --other --directory >> exclude.txt
@@ -71,7 +85,7 @@ $SED 's/recipe.hooks.core.prebuild.1.pattern.*//g' \
  > $outdir/platform.txt
 
 # Put core version and short hash of git version into core_version.h
-ver_define=`echo $ver | tr "[:lower:].-" "[:upper:]_"`
+ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"`
 echo Ver define: $ver_define
 echo \#define ARDUINO_ESP8266_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp8266/core_version.h
 echo \#define ARDUINO_ESP8266_RELEASE_$ver_define >>$outdir/cores/esp8266/core_version.h
@@ -90,14 +104,34 @@ echo Size: $size
 echo SHA-256: $sha
 
 echo "Making package_esp8266com_index.json"
-cat $srcdir/package/package_esp8266com_index.template.json | \
-jq ".packages[0].platforms[0].version = \"$ver\" | \
+
+jq_arg=".packages[0].platforms[0].version = \"$ver\" | \
     .packages[0].platforms[0].url = \"$PKG_URL\" |\
-    .packages[0].platforms[0].archiveFileName = \"$package_name.zip\" |\
-    .packages[0].platforms[0].checksum = \"SHA-256:$sha\" |\
-    .packages[0].platforms[0].size = \"$size\" |\
-    .packages[0].platforms[0].help.online = \"$DOC_URL\"" \
-    > package_esp8266com_index.json
+    .packages[0].platforms[0].archiveFileName = \"$package_name.zip\""
+
+if [ -z "$is_nightly" ]; then
+    jq_arg="$jq_arg |\
+        .packages[0].platforms[0].size = \"$size\" |\
+        .packages[0].platforms[0].checksum = \"SHA-256:$sha\" |"
+fi
+
+if [ ! -z "$DOC_URL" ]; then
+    jq_arg="$jq_arg |\
+        .packages[0].platforms[0].help.online = \"$DOC_URL\""
+fi
+
+cat $srcdir/package/package_esp8266com_index.template.json | \
+    jq "$jq_arg" > package_esp8266com_index.json
+
+old_json=package_esp8266com_index_stable.json
+for i in $(seq 1 5); do
+    echo "Downloading old package, try $i"
+    curl -L -o $old_json http://arduino.esp8266.com/stable/package_esp8266com_index.json && break
+done
+new_json=package_esp8266com_index.json
+
+set +e
+python ../../merge_packages.py $new_json $old_json >tmp && mv tmp $new_json && rm $old_json
 
 popd
 popd

package/deploy_nightly_package.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+export PKG_URL_PREFIX=https://dl.bintray.com/igrr/arduino-esp8266
+commit=`git rev-parse --short HEAD`
+
+./build_boards_manager_package.sh
+
+ver=`ls -1 versions`
+bintray_slug=igrr/arduino-esp8266/arduino-esp8266-core
+
+# Upload to bintray
+# URL to the file will look like this: https://dl.bintray.com/igrr/arduino-esp8266/:esp8266-2.4.0-nightly+20170218.zip
+curl --progress-bar \
+	-T versions/$ver/esp8266-$ver.zip \
+	-uigrr:$BINTRAY_API_KEY \
+	-o curl.out \
+	https://api.bintray.com/content/$bintray_slug/$ver/esp8266-$ver.zip
+
+
+# Publish the uploaded file
+curl -uigrr:$BINTRAY_API_KEY \
+	-X POST \
+	https://api.bintray.com/content/$bintray_slug/$ver/publish
+
+# Load deploy key
+echo -n $ESP8266_ARDUINO_DEPLOY_KEY_B64 > ~/.ssh/esp8266_arduino_deploy_b64
+base64 --decode --ignore-garbage ~/.ssh/esp8266_arduino_deploy_b64 > ~/.ssh/esp8266_arduino_deploy
+chmod 600 ~/.ssh/esp8266_arduino_deploy
+echo -e "Host $DEPLOY_HOST_NAME\n\tUser $DEPLOY_USER_NAME\n\tStrictHostKeyChecking no\n\tIdentityFile ~/.ssh/esp8266_arduino_deploy" >> ~/.ssh/config
+
+# Generate the badge (used in Readme.md)
+release_date=$(date "+%b_%d,_%Y")
+curl -o versions/$ver/badge.svg https://img.shields.io/badge/updated-$release_date-blue.svg
+
+# Check old version
+oldver=$(ssh $DEPLOY_HOST_NAME "cat $DEPLOY_PATH_NIGHTLY/version")
+
+if [ "$oldver" = "$ver" ]; then
+	echo "Nightly version hasn't changed, not updating"
+else
+	# Upload new json, version file, and the badge
+	echo $ver > versions/$ver/version
+	scp versions/$ver/{package_esp8266com_index.json,version,badge.svg} $DEPLOY_HOST_NAME:$DEPLOY_PATH_NIGHTLY/
+	curl --data-urlencode "message=Updating **nightly** package from $oldver to $ver" $GITTER_WEBHOOK
+fi

package/package_esp8266com_index.template.json

@@ -16,8 +16,6 @@
           "category": "ESP8266",
           "url": "",
           "archiveFileName": "",
-          "checksum": "",
-          "size": "",
           "help": {
             "online": ""
           },

platform.txt

@@ -6,7 +6,7 @@
 # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5---3rd-party-Hardware-specification
 
 name=ESP8266 Modules
-version=2.3.0
+version=2.4.0
 
 runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf
 runtime.tools.esptool.path={runtime.platform.path}/tools/esptool

tests/common.sh

@@ -84,3 +84,67 @@ function install_libraries()
 
     popd
 }
+
+function install_ide()
+{
+    local ide_path=$1
+    local core_path=$2
+    wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
+    tar xf arduino.tar.xz
+    mv arduino-nightly $ide_path
+    cd $ide_path/hardware
+    mkdir esp8266com
+    cd esp8266com
+    ln -s $core_path esp8266
+    cd esp8266/tools
+    python get.py
+    export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+}
+
+function run_host_tests()
+{
+    pushd host
+    make
+    make clean-objects
+    popd
+}
+
+
+function run_travis_ci_build()
+{
+    export CXX="g++-4.8" CC="gcc-4.8" GCOV="gcov-4.8"
+    echo -e "travis_fold:start:host_tests"
+    cd $TRAVIS_BUILD_DIR/tests
+    run_host_tests
+    echo -e "travis_fold:end:host_tests"
+    echo -e "travis_fold:start:sketch_test_env_prepare"
+    cd $TRAVIS_BUILD_DIR
+    install_ide $HOME/arduino_ide $TRAVIS_BUILD_DIR
+    which arduino
+    cd $TRAVIS_BUILD_DIR
+    install_libraries
+    echo -e "travis_fold:end:sketch_test_env_prepare"
+    echo -e "travis_fold:start:sketch_test"
+    build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries"
+    echo -e "travis_fold:end:sketch_test"
+    echo -e "travis_fold:start:size_report"
+    cat size.log
+    echo -e "travis_fold:end:size_report"
+    pushd $TRAVIS_BUILD_DIR/tests/host
+    bash <(curl -s https://codecov.io/bash) -X gcov
+}
+
+function deploy_nightly_package()
+{
+    cd $TRAVIS_BUILD_DIR/package
+    ./deploy_nightly_package.sh
+}
+
+set -e
+
+if [ "$BUILD_TYPE" = "deploy_nightly_package" ]; then
+    deploy_nightly_package
+elif [ "$BUILD_TYPE" = "build" ]; then
+    run_travis_ci_build
+fi
+