Initial support for Python 3.5+ // Resolve platformio#895 Resolve platformio#1365

Author: ivankravets

.appveyor.yml

@@ -7,6 +7,8 @@ platform:
 environment:
   matrix:
     - TOXENV: "py27"
+    - TOXENV: "py35"
+    - TOXENV: "py36"
 
 install:
   - cmd: git submodule update --init --recursive

.pylintrc

@@ -20,4 +20,4 @@ confidence=
 # --disable=W"
 # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
 
-disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens
+disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias

.travis.yml

@@ -6,14 +6,18 @@ matrix:
       sudo: false
       python: 2.7
       env: TOX_ENV=docs
-    - os: linux
-      sudo: false
-      python: 2.7
-      env: TOX_ENV=lint
     - os: linux
       sudo: required
       python: 2.7
       env: TOX_ENV=py27
+    - os: linux
+      sudo: required
+      python: 3.5
+      env: TOX_ENV=py35
+    - os: linux
+      sudo: required
+      python: 3.6
+      env: TOX_ENV=py36
     - os: osx
       language: generic
       env: TOX_ENV=skipexamples
@@ -24,7 +28,7 @@ install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi
 
   # ChipKIT issue: install 32-bit support for GCC PIC32
-  - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
 
 script:
   - tox -e $TOX_ENV

HISTORY.rst

@@ -1,10 +1,19 @@
 Release Notes
 =============
 
+PlatformIO 4.0
+--------------
+
+4.0.0 (2019-??-??)
+~~~~~~~~~~~~~~~~~~
+
+* Added Python 3.5+ support
+  (`issue #895 <https://github.com/platformio/platformio-core/issues/895>`_)
+
 PlatformIO 3.0
 --------------
 
-3.6.4 (2018-??-??)
+3.6.4 (2019-??-??)
 ~~~~~~~~~~~~~~~~~~
 
 * Improved Project Generator for IDEs:

platformio/__init__.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
-VERSION = (3, 6, "4b1")
+VERSION = (4, 0, "0a1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -33,10 +31,3 @@
 __copyright__ = "Copyright 2014-present PlatformIO"
 
 __apiurl__ = "https://api.platformio.org"
-
-if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0):
-    msg = ("PlatformIO Core v%s does not run under Python version %s.\n"
-           "Minimum supported version is 2.7, please upgrade Python.\n"
-           "Python 3 is not yet supported.\n")
-    sys.stderr.write(msg % (__version__, sys.version))
-    sys.exit(1)

platformio/__main__.py

@@ -53,7 +53,7 @@ def _handle_obsolate_command(name):
         if name == "platforms":
             from platformio.commands import platform
             return platform.cli
-        elif name == "serialports":
+        if name == "serialports":
             from platformio.commands import device
             return device.cli
         raise AttributeError()

platformio/app.py

@@ -169,8 +169,11 @@ def get_cache_path(self, key):
     @staticmethod
     def key_from_args(*args):
         h = hashlib.md5()
-        for data in args:
-            h.update(str(data))
+        for arg in args:
+            if not arg:
+                continue
+            arg = str(arg)
+            h.update(arg if util.PY2 else arg.encode())
         return h.hexdigest()
 
     def get(self, key):
@@ -191,7 +194,7 @@ def set(self, key, data, valid):
         if not isdir(self.cache_dir):
             os.makedirs(self.cache_dir)
         tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
-        assert valid.endswith(tuple(tdmap.keys()))
+        assert valid.endswith(tuple(tdmap))
         expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
 
         if not self._lock_dbindex():
@@ -339,21 +342,22 @@ def is_disabled_progressbar():
 
 def get_cid():
     cid = get_state_item("cid")
-    if not cid:
-        _uid = None
-        if getenv("C9_UID"):
-            _uid = getenv("C9_UID")
-        elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
-            try:
-                _uid = requests.get("{api}/user?token={token}".format(
-                    api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
-                    token=getenv("USER_TOKEN"))).json().get("id")
-            except:  # pylint: disable=bare-except
-                pass
-        cid = str(
-            uuid.UUID(
-                bytes=hashlib.md5(str(_uid if _uid else uuid.getnode())).
-                digest()))
-        if "windows" in util.get_systype() or os.getuid() > 0:
-            set_state_item("cid", cid)
+    if cid:
+        return cid
+    uid = None
+    if getenv("C9_UID"):
+        uid = getenv("C9_UID")
+    elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
+        try:
+            uid = requests.get("{api}/user?token={token}".format(
+                api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
+                token=getenv("USER_TOKEN"))).json().get("id")
+        except:  # pylint: disable=bare-except
+            pass
+    if not uid:
+        uid = uuid.getnode()
+    cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest())
+    cid = str(cid)
+    if "windows" in util.get_systype() or os.getuid() > 0:
+        set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -92,7 +92,7 @@
     variables=commonvars,
 
     # Propagating External Environment
-    PIOVARIABLES=commonvars.keys(),
+    PIOVARIABLES=list(commonvars.keys()),
     ENV=environ,
     UNIX_TIME=int(time()),
     PIOHOME_DIR=util.get_home_dir(),
@@ -125,9 +125,11 @@
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 
 # decode common variables
-for k in commonvars.keys():
+for k in list(commonvars.keys()):
     if k in env:
         env[k] = base64.b64decode(env[k])
+        if isinstance(env[k], bytes):
+            env[k] = env[k].decode()
         if k in MULTILINE_VARS:
             env[k] = util.parse_conf_multi_values(env[k])
 
@@ -161,7 +163,9 @@
 env.LoadPioPlatform(commonvars)
 
 env.SConscriptChdir(0)
-env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite"))
+env.SConsignFile(
+    join("$PROJECTBUILD_DIR",
+         ".sconsign.dblite" if util.PY2 else ".sconsign3.dblite"))
 
 for item in env.GetExtraScripts("pre"):
     env.SConscript(item, exports="env")

platformio/builder/tools/piolib.py

@@ -74,12 +74,19 @@ def get_used_frameworks(env, path):
                 if not env.IsFileWithExt(
                         fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT):
                     continue
-                with open(join(root, fname)) as f:
-                    content = f.read()
-                    if "Arduino.h" in content and include_re.search(content):
-                        return ["arduino"]
-                    elif "mbed.h" in content and include_re.search(content):
-                        return ["mbed"]
+                content = ""
+                try:
+                    with open(join(root, fname)) as f:
+                        content = f.read()
+                except UnicodeDecodeError:
+                    with open(join(root, fname), encoding="latin-1") as f:
+                        content = f.read()
+                if not content:
+                    continue
+                if "Arduino.h" in content and include_re.search(content):
+                    return ["arduino"]
+                if "mbed.h" in content and include_re.search(content):
+                    return ["mbed"]
         return []
 
 
@@ -183,9 +190,9 @@ def get_include_dirs(self):
 
     @property
     def build_dir(self):
-        return join("$BUILD_DIR",
-                    "lib%s" % hashlib.sha1(self.path).hexdigest()[:3],
-                    basename(self.path))
+        lib_hash = hashlib.sha1(self.path if util.PY2 else self.path.
+                                encode()).hexdigest()[:3]
+        return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path))
 
     @property
     def build_flags(self):
@@ -227,7 +234,7 @@ def is_built(self):
 
     @staticmethod
     def validate_ldf_mode(mode):
-        if isinstance(mode, basestring):
+        if isinstance(mode, util.string_types):
             mode = mode.strip().lower()
         if mode in LibBuilderBase.LDF_MODES:
             return mode
@@ -239,7 +246,7 @@ def validate_ldf_mode(mode):
 
     @staticmethod
     def validate_compat_mode(mode):
-        if isinstance(mode, basestring):
+        if isinstance(mode, util.string_types):
             mode = mode.strip().lower()
         if mode in LibBuilderBase.COMPAT_MODES:
             return mode
@@ -612,9 +619,9 @@ def _is_arduino_manifest(self):
     def src_filter(self):
         if "srcFilter" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("srcFilter")
-        elif self.env['SRC_FILTER']:
+        if self.env['SRC_FILTER']:
             return self.env['SRC_FILTER']
-        elif self._is_arduino_manifest():
+        if self._is_arduino_manifest():
             return ArduinoLibBuilder.src_filter.fget(self)
         return LibBuilderBase.src_filter.fget(self)
 

platformio/builder/tools/piomisc.py

@@ -162,7 +162,7 @@ def append_prototypes(self, contents):
         if not prototypes:
             return contents
 
-        prototype_names = set([m.group(3).strip() for m in prototypes])
+        prototype_names = set(m.group(3).strip() for m in prototypes)
         split_pos = prototypes[0].start()
         match_ptrs = re.search(
             self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
@@ -212,7 +212,7 @@ def _get_compiler_type(env):
     output = "".join([result['out'], result['err']]).lower()
     if "clang" in output and "LLVM" in output:
         return "clang"
-    elif "gcc" in output:
+    if "gcc" in output:
         return "gcc"
     return None
 

platformio/builder/tools/pioplatform.py

@@ -95,19 +95,24 @@ def LoadPioPlatform(env, variables):
         for key, value in variables.UnknownVariables().items():
             if not key.startswith("BOARD_"):
                 continue
-            env.Replace(
-                **{key.upper().replace("BUILD.", ""): base64.b64decode(value)})
+            value = base64.b64decode(value)
+            if isinstance(value, bytes):
+                value = value.decode()
+            env.Replace(**{key.upper().replace("BUILD.", ""): value})
         return
 
     # update board manifest with a custom data
     board_config = env.BoardConfig()
     for key, value in variables.UnknownVariables().items():
         if not key.startswith("BOARD_"):
             continue
-        board_config.update(key.lower()[6:], base64.b64decode(value))
+        value = base64.b64decode(value)
+        if isinstance(value, bytes):
+            value = value.decode()
+        board_config.update(key.lower()[6:], value)
 
     # update default environment variables
-    for key in variables.keys():
+    for key in list(variables.keys()):
         if key in env or \
                 not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]):
             continue

platformio/builder/tools/piowinhooks.py

@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+
 from hashlib import md5
 from os import makedirs
 from os.path import isdir, isfile, join
-from platform import system
+
+from platformio import util
 
 # Windows CLI has limit with command length to 8192
 # Leave 2000 chars for flags and other options
@@ -58,7 +61,9 @@ def _file_long_data(env, data):
     build_dir = env.subst("$BUILD_DIR")
     if not isdir(build_dir):
         makedirs(build_dir)
-    tmp_file = join(build_dir, "longcmd-%s" % md5(data).hexdigest())
+    tmp_file = join(
+        build_dir,
+        "longcmd-%s" % md5(data if util.PY2 else data.encode()).hexdigest())
     if isfile(tmp_file):
         return tmp_file
     with open(tmp_file, "w") as fp:
@@ -71,7 +76,7 @@ def exists(_):
 
 
 def generate(env):
-    if system() != "Windows":
+    if "windows" not in util.get_systype():
         return None
 
     env.Replace(_long_sources_hook=long_sources_hook)

platformio/builder/tools/platformio.py

@@ -24,7 +24,7 @@
 from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
                           DefaultEnvironment, Export, SConscript)
 
-from platformio.util import glob_escape, pioversion_to_intstr
+from platformio.util import glob_escape, pioversion_to_intstr, string_types
 
 SRC_HEADER_EXT = ["h", "hpp"]
 SRC_C_EXT = ["c", "cc", "cpp"]
@@ -189,7 +189,7 @@ def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
     # provided with a -U option // Issue #191
     undefines = [
         u for u in env.get("CCFLAGS", [])
-        if isinstance(u, basestring) and u.startswith("-U")
+        if isinstance(u, string_types) and u.startswith("-U")
     ]
     if undefines:
         for undef in undefines:

platformio/commands/device.py

@@ -44,7 +44,7 @@ def device_list(  # pylint: disable=too-many-branches
     if mdns:
         data['mdns'] = util.get_mdns_services()
 
-    single_key = data.keys()[0] if len(data.keys()) == 1 else None
+    single_key = list(data)[0] if len(list(data)) == 1 else None
 
     if json_output:
         return click.echo(json.dumps(data[single_key] if single_key else data))