Standardise ensure_dir and rmdir (#2871)

* Standardise ensure_dir and rmdir

* Standardise ensure_dir and rmdir

* Add libmysqlclient to broken list

* Libtorrent failing to be rebuilt

* Add boost to broken recipes list

Author: Julian-O

ci/constants.py

@@ -33,6 +33,13 @@ class TargetPython(Enum):
     'twisted',
     # genericndkbuild is incompatible with sdl2 (which is build by default when targeting sdl2 bootstrap)
     'genericndkbuild',
+    # libmysqlclient gives a linker failure (See issue #2808)
+    'libmysqlclient',
+    # boost gives errors (requires numpy? syntax error in .jam?)
+    'boost',
+    # libtorrent gives errors (requires boost. Also, see issue #2809, to start with)
+    'libtorrent',
+
 ])
 
 BROKEN_RECIPES = {

pythonforandroid/bdistapk.py

@@ -1,10 +1,10 @@
+from glob import glob
+from os.path import realpath, join, dirname, curdir, basename, split
 from setuptools import Command
-
+from shutil import copyfile
 import sys
-from os.path import realpath, join, exists, dirname, curdir, basename, split
-from os import makedirs
-from glob import glob
-from shutil import rmtree, copyfile
+
+from pythonforandroid.util import rmdir, ensure_dir
 
 
 def argv_contains(t):
@@ -90,9 +90,8 @@ def prepare_build_dir(self):
                   'that.')
 
         bdist_dir = 'build/bdist.android-{}'.format(self.arch)
-        if exists(bdist_dir):
-            rmtree(bdist_dir)
-        makedirs(bdist_dir)
+        rmdir(bdist_dir)
+        ensure_dir(bdist_dir)
 
         globs = []
         for directory, patterns in self.distribution.package_data.items():
@@ -107,8 +106,7 @@ def prepare_build_dir(self):
         if not argv_contains('--launcher'):
             for filen in filens:
                 new_dir = join(bdist_dir, dirname(filen))
-                if not exists(new_dir):
-                    makedirs(new_dir)
+                ensure_dir(new_dir)
                 print('Including {}'.format(filen))
                 copyfile(filen, join(bdist_dir, filen))
                 if basename(filen) in ('main.py', 'main.pyc'):

pythonforandroid/bootstrap.py

@@ -10,7 +10,8 @@
 
 from pythonforandroid.logger import (shprint, info, logger, debug)
 from pythonforandroid.util import (
-    current_directory, ensure_dir, temp_directory, BuildInterruptingException)
+    current_directory, ensure_dir, temp_directory, BuildInterruptingException,
+    rmdir)
 from pythonforandroid.recipe import Recipe
 
 
@@ -396,7 +397,7 @@ def fry_eggs(self, sitepackages):
                 files = [join(rd, f) for f in listdir(rd) if f != 'EGG-INFO']
                 if files:
                     shprint(sh.mv, '-t', sitepackages, *files)
-                shprint(sh.rm, '-rf', d)
+                rmdir(d)
 
 
 def expand_dependencies(recipes, ctx):

pythonforandroid/bootstraps/common/build/build.py

@@ -21,6 +21,8 @@
 from fnmatch import fnmatch
 import jinja2
 
+from pythonforandroid.util import rmdir, ensure_dir
+
 
 def get_dist_info_for(key, error_if_missing=True):
     try:
@@ -93,11 +95,6 @@ def get_bootstrap_name():
 DEFAULT_PYTHON_SERVICE_JAVA_CLASS = 'org.kivy.android.PythonService'
 
 
-def ensure_dir(path):
-    if not exists(path):
-        makedirs(path)
-
-
 def render(template, dest, **kwargs):
     '''Using jinja2, render `template` to the filename `dest`, supplying the
 
@@ -241,7 +238,7 @@ def make_package(args):
     assets_dir = "src/main/assets"
 
     # Delete the old assets.
-    shutil.rmtree(assets_dir, ignore_errors=True)
+    rmdir(assets_dir, ignore_errors=True)
     ensure_dir(assets_dir)
 
     # Add extra environment variable file into tar-able directory:
@@ -290,7 +287,7 @@ def make_package(args):
                                     not exists(
                                         join(main_py_only_dir, dir_path)
                                     )):
-                                os.mkdir(join(main_py_only_dir, dir_path))
+                                ensure_dir(join(main_py_only_dir, dir_path))
                             # Copy actual file:
                             shutil.copyfile(
                                 join(args.private, variant),
@@ -328,17 +325,17 @@ def make_package(args):
             )
     finally:
         for directory in _temp_dirs_to_clean:
-            shutil.rmtree(directory)
+            rmdir(directory)
 
     # Remove extra env vars tar-able directory:
-    shutil.rmtree(env_vars_tarpath)
+    rmdir(env_vars_tarpath)
 
     # Prepare some variables for templating process
     res_dir = "src/main/res"
     res_dir_initial = "src/res_initial"
     # make res_dir stateless
     if exists(res_dir_initial):
-        shutil.rmtree(res_dir, ignore_errors=True)
+        rmdir(res_dir, ignore_errors=True)
         shutil.copytree(res_dir_initial, res_dir)
     else:
         shutil.copytree(res_dir, res_dir_initial)

pythonforandroid/bootstraps/sdl2/__init__.py

@@ -1,9 +1,11 @@
-from pythonforandroid.toolchain import (
-    Bootstrap, shprint, current_directory, info, info_main)
-from pythonforandroid.util import ensure_dir
 from os.path import join
+
 import sh
 
+from pythonforandroid.toolchain import (
+    Bootstrap, shprint, current_directory, info, info_main)
+from pythonforandroid.util import ensure_dir, rmdir
+
 
 class SDL2GradleBootstrap(Bootstrap):
     name = 'sdl2'
@@ -15,8 +17,8 @@ class SDL2GradleBootstrap(Bootstrap):
     def assemble_distribution(self):
         info_main("# Creating Android project ({})".format(self.name))
 
+        rmdir(self.dist_dir)
         info("Copying SDL2/gradle build")
-        shprint(sh.rm, "-rf", self.dist_dir)
         shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
 
         # either the build use environment variable (ANDROID_HOME)

pythonforandroid/bootstraps/service_only/__init__.py

@@ -2,7 +2,7 @@
 from os.path import join
 from pythonforandroid.toolchain import (
     Bootstrap, current_directory, info, info_main, shprint)
-from pythonforandroid.util import ensure_dir
+from pythonforandroid.util import ensure_dir, rmdir
 
 
 class ServiceOnlyBootstrap(Bootstrap):
@@ -18,7 +18,7 @@ def assemble_distribution(self):
             self.name))
 
         info('This currently just copies the build stuff straight from the build dir.')
-        shprint(sh.rm, '-rf', self.dist_dir)
+        rmdir(self.dist_dir)
         shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
         with current_directory(self.dist_dir):
             with open('local.properties', 'w') as fileh:

pythonforandroid/bootstraps/webview/__init__.py

@@ -1,8 +1,10 @@
-from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint
-from pythonforandroid.util import ensure_dir
 from os.path import join
+
 import sh
 
+from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint
+from pythonforandroid.util import ensure_dir, rmdir
+
 
 class WebViewBootstrap(Bootstrap):
     name = 'webview'
@@ -15,7 +17,7 @@ def assemble_distribution(self):
         info_main('# Creating Android project from build and {} bootstrap'.format(
             self.name))
 
-        shprint(sh.rm, '-rf', self.dist_dir)
+        rmdir(self.dist_dir)
         shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
         with current_directory(self.dist_dir):
             with open('local.properties', 'w') as fileh:

pythonforandroid/build.py

@@ -1,28 +1,29 @@
+from contextlib import suppress
+import copy
+import glob
+import os
+from os import environ
 from os.path import (
     abspath, join, realpath, dirname, expanduser, exists
 )
-from os import environ
-import copy
-import os
-import glob
 import re
-import sh
 import shutil
 import subprocess
-from contextlib import suppress
 
-from pythonforandroid.util import (
-    current_directory, ensure_dir,
-    BuildInterruptingException,
-)
-from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)
+import sh
+
+from pythonforandroid.androidndk import AndroidNDK
 from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
+from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)
 from pythonforandroid.pythonpackage import get_package_name
 from pythonforandroid.recipe import CythonRecipe, Recipe
 from pythonforandroid.recommendations import (
     check_ndk_version, check_target_api, check_ndk_api,
     RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
-from pythonforandroid.androidndk import AndroidNDK
+from pythonforandroid.util import (
+    current_directory, ensure_dir,
+    BuildInterruptingException, rmdir
+)
 
 
 def get_targets(sdk_dir):
@@ -641,7 +642,7 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None):
             for f in set(copied_over_contents + new_venv_additions):
                 full_path = os.path.join(venv_site_packages_dir, f)
                 if os.path.isdir(full_path):
-                    shutil.rmtree(full_path)
+                    rmdir(full_path)
                 else:
                     os.remove(full_path)
         finally:

pythonforandroid/distribution.py

@@ -1,10 +1,11 @@
-from os.path import exists, join
-import glob
 import json
+import glob
+from os.path import exists, join
 
-from pythonforandroid.logger import (debug, info, info_notify, warning, Err_Style, Err_Fore)
-from pythonforandroid.util import current_directory, BuildInterruptingException
-from shutil import rmtree
+from pythonforandroid.logger import (
+    debug, info, info_notify, warning, Err_Style, Err_Fore)
+from pythonforandroid.util import (
+    current_directory, BuildInterruptingException, rmdir)
 
 
 class Distribution:
@@ -201,7 +202,7 @@ def folder_exists(self):
         return exists(self.dist_dir)
 
     def delete(self):
-        rmtree(self.dist_dir)
+        rmdir(self.dist_dir)
 
     @classmethod
     def get_distributions(cls, ctx, extra_dist_dirs=[]):

pythonforandroid/prerequisites.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
-import sys
-import platform
 import os
-import subprocess
+import platform
 import shutil
+import subprocess
+import sys
+
 from pythonforandroid.logger import info, warning, error
+from pythonforandroid.util import ensure_dir
 
 
 class Prerequisite(object):
@@ -247,13 +249,7 @@ def darwin_installer(self):
                 "~/Library/Java/JavaVirtualMachines"
             )
             info(f"Extracting {filename} to {user_library_java_path}")
-            subprocess.check_output(
-                [
-                    "mkdir",
-                    "-p",
-                    user_library_java_path,
-                ],
-            )
+            ensure_dir(user_library_java_path)
             subprocess.check_output(
                 ["tar", "xzf", f"/tmp/{filename}", "-C", user_library_java_path],
             )

pythonforandroid/pythonpackage.py

@@ -34,21 +34,23 @@
 
 
 import functools
+from io import open  # needed for python 2
 import os
 import shutil
 import subprocess
 import sys
 import tarfile
 import tempfile
 import time
-import zipfile
-from io import open  # needed for python 2
 from urllib.parse import unquote as urlunquote
 from urllib.parse import urlparse
+import zipfile
 
 import toml
 import build.util
 
+from pythonforandroid.util import rmdir, ensure_dir
+
 
 def transform_dep_for_pip(dependency):
     if dependency.find("@") > 0 and (
@@ -113,7 +115,7 @@ def extract_metainfo_files_from_package(
 
         _extract_metainfo_files_from_package_unsafe(package, output_folder)
     finally:
-        shutil.rmtree(temp_folder)
+        rmdir(temp_folder)
 
 
 def _get_system_python_executable():
@@ -314,7 +316,7 @@ def get_package_as_folder(dependency):
             )
 
         # Create download subfolder:
-        os.mkdir(os.path.join(venv_path, "download"))
+        ensure_dir(os.path.join(venv_path, "download"))
 
         # Write a requirements.txt with our package and download:
         with open(os.path.join(venv_path, "requirements.txt"),
@@ -394,11 +396,11 @@ def to_unicode(s):  # Needed for Python 2.
         # Copy result to new dedicated folder so we can throw away
         # our entire virtualenv nonsense after returning:
         result_path = tempfile.mkdtemp()
-        shutil.rmtree(result_path)
+        rmdir(result_path)
         shutil.copytree(result_folder_or_file, result_path)
         return (dl_type, result_path)
     finally:
-        shutil.rmtree(venv_parent)
+        rmdir(venv_parent)
 
 
 def _extract_metainfo_files_from_package_unsafe(
@@ -458,7 +460,7 @@ def _extract_metainfo_files_from_package_unsafe(
         shutil.copyfile(metadata_path, os.path.join(output_path, "METADATA"))
     finally:
         if clean_up_path:
-            shutil.rmtree(path)
+            rmdir(path)
 
 
 def is_filesystem_path(dep):
@@ -576,7 +578,7 @@ def _extract_info_from_package(dependency,
 
             return list(set(requirements))  # remove duplicates
     finally:
-        shutil.rmtree(output_folder)
+        rmdir(output_folder)
 
 
 package_name_cache = dict()