AAB support related changes (#2467)

* Added support for aab

* Move to build:gradle:3.5.4 (adds support for API 30), fix some tests

* Github actions test apps (apk + aab)

* Add missing bdistaab

* Fix automated tests

* ndk lib folder (or ndk platform) now is ABI specific

* Fixes dist lookup + some tests

* Added .aab and .apks to blacklist

* Interrupt build and alert the user if tried to build an aab in debug mode

* Updates troubleshooting instructions to reflect current structure.

* Exclude gdbserver and gdb.setup from release builds

* Add a paragraph in history + fixes --arch docs

* Fix versioning

* Minor fixes to docs

* Some code cleanup

* Removes unusued versioning logic in unpackPyBundle and add a FIXME

Author: misl6

.github/workflows/push.yml

@@ -41,14 +41,10 @@ jobs:
         pip install tox>=2.0
         make test
 
-  build:
+  build_apk:
     name: Unit test apk
     needs: [flake8]
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        build-arch: ['arm64-v8a', 'armeabi-v7a', 'x86_64', 'x86']
     steps:
     - name: Checkout python-for-android
       uses: actions/checkout@v2
@@ -64,15 +60,43 @@ jobs:
     - name: Pull docker image
       run: |
         make docker/pull
-    - name: Build apk Python 3 ${{ matrix.build-arch }}
+    - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
       run: |
         mkdir -p apks
-        make docker/run/make/with-artifact/testapps-with-numpy/${{ matrix.build-arch }}
+        make docker/run/make/with-artifact/apk/testapps-with-numpy
     - uses: actions/upload-artifact@v1
       with:
-        name: bdist_test_app_unittests__${{ matrix.build-arch }}-debug-1.1.apk
+        name: bdist_unit_tests_app-debug-1.1-.apk
         path: apks
 
+  build_aab:
+    name: Unit test aab
+    needs: [flake8]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout python-for-android
+      uses: actions/checkout@v2
+    # helps with GitHub runner getting out of space
+    - name: Free disk space
+      run: |
+        df -h
+        sudo swapoff -a
+        sudo rm -f /swapfile
+        sudo apt -y clean
+        docker rmi $(docker image ls -aq)
+        df -h
+    - name: Pull docker image
+      run: |
+        make docker/pull
+    - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
+      run: |
+        mkdir -p aabs
+        make docker/run/make/with-artifact/aab/testapps-with-numpy-aab
+    - uses: actions/upload-artifact@v1
+      with:
+        name: bdist_unit_tests_app-release-1.1-.aab
+        path: aabs
+
   rebuild_updated_recipes:
     name: Test updated recipes
     needs: [flake8]

Makefile

@@ -34,12 +34,17 @@ rebuild_updated_recipes: virtualenv
 	ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \
 	$(PYTHON) ci/rebuild_updated_recipes.py
 
-testapps-with-numpy/%: virtualenv
-	$(eval $@_APP_ARCH := $(shell basename $*))
+testapps-with-numpy: virtualenv
 	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
     python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
     --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \
-    --arch=$($@_APP_ARCH)
+    --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86
+
+testapps-with-numpy-aab: virtualenv
+	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
+    python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \
+    --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release
 
 testapps/%: virtualenv
 	$(eval $@_APP_ARCH := $(shell basename $*))
@@ -69,14 +74,18 @@ docker/run/test: docker/build
 docker/run/command: docker/build
 	docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)"
 
-docker/run/make/%: docker/build
-	docker run --rm --env-file=.env $(DOCKER_IMAGE) make $*
+docker/run/make/with-artifact/apk/%: docker/build
+	docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
+	docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1-.apk ./apks
+	docker rm -fv p4a-latest
 
-docker/run/make/with-artifact/%: docker/build
-	$(eval $@_APP_ARCH := $(shell basename $*))
+docker/run/make/with-artifact/aab/%: docker/build
 	docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
-	docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app__$($@_APP_ARCH)-debug-1.1-.apk ./apks
+	docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1-.aab ./aabs
 	docker rm -fv p4a-latest
 
+docker/run/make/%: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE) make $*
+
 docker/run/shell: docker/build
 	docker run --rm --env-file=.env -it $(DOCKER_IMAGE)

README.md

@@ -8,7 +8,7 @@ python-for-android
 
 python-for-android is a packaging tool for Python apps on Android. You can
 create your own Python distribution including the modules and
-dependencies you want, and bundle it in an APK along with your own code.
+dependencies you want, and bundle it in an APK or AAB along with your own code.
 
 Features include:
 
@@ -19,6 +19,7 @@ Features include:
    sqlalchemy.
 -  Multiple architecture targets, for APKs optimised on any given
    device.
+-  AAB: Android App Bundle support.
 
 For documentation and support, see:
 
@@ -30,7 +31,7 @@ For documentation and support, see:
 
 Follow the [quickstart
 instructions](<https://python-for-android.readthedocs.org/en/latest/quickstart/>)
-to install and begin creating APKs.
+to install and begin creating APKs and AABs.
 
 **Quick instructions**: install python-for-android with:
 
@@ -52,6 +53,8 @@ With everything installed, build an APK with SDL2 with e.g.:
 
     p4a apk --requirements=kivy --private /home/username/devel/planewave_frozen/ --package=net.inclem.planewavessdl2 --name="planewavessdl2" --version=0.5 --bootstrap=sdl2
 
+**If you need to deploy your app on Google Play, Android App Bundle (aab) is required since 1 August 2021:**
+
 **For full instructions and parameter options,** see [the
 documentation](https://python-for-android.readthedocs.io/en/latest/quickstart/#usage).
 
@@ -109,6 +112,9 @@ api level below 21, you should use an older version of python-for-android
 On March of 2020 we dropped support for creating apps that use Python 2. The latest
 python-for-android release that supported building Python 2 was version 2019.10.6.
 
+On August of 2021, we added support for Android App Bundle (aab). As a collateral,
+now We support multi-arch apk.
+
 ## Contributors
 
 This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].

doc/source/commands.rst

@@ -71,8 +71,9 @@ supply those that you need.
   Whether the distribution must be compiled from scratch.
 
 ``--arch``
-  The architecture to build for. Currently only one architecture can be 
-  targeted at a time, and a given distribution can only include one architecture.
+  The architecture to build for. You can specify multiple architectures to build for
+  at the same time. As an example ``p4a ... --arch arm64-v8a --arch armeabi-v7a ...``
+  will build a distribution for both ``arm64-v8a`` and ``armeabi-v7a``.
 
 ``--bootstrap BOOTSTRAP``
   The Java bootstrap to use for your application. You mostly don't

doc/source/quickstart.rst

@@ -213,6 +213,24 @@ You can also replace flask with another web framework.
 Replace ``--port=5000`` with the port on which your app will serve a
 website. The default for Flask is 5000.
 
+Exporting the Android App Bundle (aab) for distributing it on Google Play
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Starting from August 2021 for new apps and from November 2021 for updates to existings apps,
+Google Play Console will require the Android App Bundle instead of the long lived apk.
+
+python-for-android handles by itself the needed work to accomplish the new requirements:
+
+    p4a aab --private $HOME/code/myapp --package=org.example.myapp --name="My App" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy --arch=arm64-v8a --arch=armeabi-v7a --release
+
+This `p4a aab ...` command builds a distribution with `python3`,
+`kivy`, and everything else you specified in the requirements.
+It will be packaged using a SDL2 bootstrap, and produce
+an `.aab` file that contains binaries for both `armeabi-v7a` and `arm64-v8a` ABIs.
+
+The Android App Bundle, is supposed to be used for distributing your app.
+If you need to test it locally, on your device, you can use `bundletool <https://developer.android.com/studio/command-line/bundletool>`
+
 Other options
 ~~~~~~~~~~~~~
 

doc/source/troubleshooting.rst

@@ -85,31 +85,35 @@ At the top level, this will always contain the same set of files::
   AndroidManifest.xml  classes.dex  META-INF     res
   assets               lib          YourApk.apk  resources.arsc
 
-The Python distribution is in the assets folder::
+The user app data (code, images, fonts ..) is packaged into a single tarball contained in the assets folder::
 
   $ cd assets
   $ ls
-  private.mp3
+  private.tar
 
-``private.mp3`` is actually a tarball containing all your packaged
-data, and the Python distribution. Extract it::
+``private.tar`` is a tarball containing all your packaged
+data. Extract it::
 
-  $ tar xf private.mp3
+  $ tar xf private.tar
 
-This will reveal all the Python-related files::
+This will reveal all the user app data (the files shown below are from the touchtracer demo)::
 
   $ ls
-  android_runnable.pyo  include          interpreter_subprocess  main.kv   pipinterface.kv   settings.pyo
-  assets                __init__.pyo     interpreterwrapper.pyo  main.pyo  pipinterface.pyo  utils.pyo
-  editor.kv             interpreter.kv   _python_bundle          menu.kv   private.mp3       widgets.pyo
-  editor.pyo            interpreter.pyo  libpymodules.so         menu.pyo  settings.kv
+  README.txt		android.txt		icon.png		main.pyc		p4a_env_vars.txt	particle.png
+  private.tar		touchtracer.kv
 
-Most of these files have been included by the user (in this case, they
-come from one of my own apps), the rest relate to the python
-distribution.
+Due to how We're required to ship ABI-specific things in Android App Bundle,
+the Python installation is packaged separately, as (most of it) is ABI-specific.
+
+For example, the Python installation for ``arm64-v8a`` is available in ``lib/arm64-v8a/libpybundle.so``
+
+``libpybundle.so`` is a tarball (but named like a library for packaging requirements), that contains our ``_python_bundle``::
+
+  $ tar xf libpybundle.so
+  $ cd _python_bundle
+  $ ls
+  modules		site-packages	stdlib.zip
 
-The python installation, along with all side-packages, is mostly contained
-inside the `_python_bundle` folder.
 
 
 Common errors

pythonforandroid/archs.py

@@ -1,9 +1,10 @@
 from distutils.spawn import find_executable
 from os import environ
-from os.path import join, split
+from os.path import join, split, exists
 from multiprocessing import cpu_count
 from glob import glob
 
+from pythonforandroid.logger import warning
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.util import BuildInterruptingException, build_platform
 
@@ -30,7 +31,7 @@ class Arch:
     common_cppflags = [
         '-DANDROID',
         '-D__ANDROID_API__={ctx.ndk_api}',
-        '-I{ctx.ndk_dir}/sysroot/usr/include/{command_prefix}',
+        '-I{ctx.ndk_sysroot}/usr/include/{command_prefix}',
         '-I{python_includes}',
     ]
 
@@ -57,6 +58,24 @@ def __init__(self, ctx):
     def __str__(self):
         return self.arch
 
+    @property
+    def ndk_lib_dir(self):
+        return join(self.ctx.ndk_sysroot, 'usr', 'lib', self.command_prefix, str(self.ctx.ndk_api))
+
+    @property
+    def ndk_platform(self):
+        warning("ndk_platform is deprecated and should be avoided in new recipes")
+        ndk_platform = join(
+            self.ctx.ndk_dir,
+            'platforms',
+            'android-{}'.format(self.ctx.ndk_api),
+            self.platform_dir)
+        if not exists(ndk_platform):
+            BuildInterruptingException(
+                "The requested platform folder doesn't exist. If you're building on ndk >= r22, and seeing this error, one of the required recipe is using a removed feature."
+            )
+        return ndk_platform
+
     @property
     def include_dirs(self):
         return [
@@ -133,7 +152,7 @@ def get_env(self, with_flags_in_cc=True):
             ctx=self.ctx,
             command_prefix=self.command_prefix,
             python_includes=join(
-                self.ctx.get_python_install_dir(),
+                self.ctx.get_python_install_dir(self.arch),
                 'include/python{}'.format(self.ctx.python_recipe.version[0:3]),
             ),
         )
@@ -213,7 +232,7 @@ def get_env(self, with_flags_in_cc=True):
         # Android's arch/toolchain
         env['ARCH'] = self.arch
         env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
-        env['TOOLCHAIN_PREFIX'] = self.ctx.toolchain_prefix
+        env['TOOLCHAIN_PREFIX'] = self.toolchain_prefix
         env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version
 
         # Custom linker options

pythonforandroid/bdistapk.py

@@ -128,21 +128,23 @@ def prepare_build_dir(self):
 
 
 class BdistAPK(Bdist):
-    """
-    distutil command handler for 'apk'
-    """
+    """distutil command handler for 'apk'."""
     description = 'Create an APK with python-for-android'
     package_type = 'apk'
 
 
 class BdistAAR(Bdist):
-    """
-    distutil command handler for 'aar'
-    """
+    """distutil command handler for 'aar'."""
     description = 'Create an AAR with python-for-android'
     package_type = 'aar'
 
 
+class BdistAAB(Bdist):
+    """distutil command handler for 'aab'."""
+    description = 'Create an AAB with python-for-android'
+    package_type = 'aab'
+
+
 def _set_user_options():
     # This seems like a silly way to do things, but not sure if there's a
     # better way to pass arbitrary options onwards to p4a
@@ -156,6 +158,7 @@ def _set_user_options():
                 user_options.append((arg[2:], None, None))
 
     BdistAPK.user_options = user_options
+    BdistAAB.user_options = user_options
 
 
 _set_user_options()

pythonforandroid/bootstrap.py

@@ -374,7 +374,7 @@ def strip_libraries(self, arch):
         if len(tokens) > 1:
             strip = strip.bake(tokens[1:])
 
-        libs_dir = join(self.dist_dir, '_python_bundle',
+        libs_dir = join(self.dist_dir, f'_python_bundle__{arch.arch}',
                         '_python_bundle', 'modules')
         filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'),
                          '-iname', '*.so', _env=env).stdout.decode('utf-8')