Fix part of #1719, part of #3709: Add build stats CI workflow

.github/workflows/stats.yml

@@ -0,0 +1,216 @@
+# Contains jobs corresponding to stats, including build stats due to changes in a PR.
+
+name: Stats Checks & Reports
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 02 * * *"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  find_open_pull_requests:
+    name: Find open PRs
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.compute-pull-request-matrix.outputs.matrix }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute PR matrix
+        id: compute-pull-request-matrix
+        # Remove spaces to ensure the matrix output is on one line. Reference:
+        # https://stackoverflow.com/a/3232433.
+        run: |
+          CURRENT_OPEN_PR_INFO="$(gh pr list --json number,baseRefName,headRefName,headRepository,headRepositoryOwner | tr -d '[:space:]')"
+          echo "matrix={\"prInfo\": $CURRENT_OPEN_PR_INFO}" >> "$GITHUB_OUTPUT"
+
+  build_stats:
+    name: Build Stats
+    needs: find_open_pull_requests
+    runs-on: ubuntu-20.04
+    # Reduce parallelization due to high build times, and allow individual PRs to fail.
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix: ${{ fromJson(needs.find_open_pull_requests.outputs.matrix) }}
+    env:
+      ENABLE_CACHING: false
+      CACHE_DIRECTORY: ~/.bazel_cache
+    steps:
+      - name: Compute PR head owner/repo reference
+        env:
+          PR_HEAD_REPO: ${{ matrix.prInfo.headRepository.name }}
+          PR_HEAD_REPO_OWNER: ${{ matrix.prInfo.headRepositoryOwner.login }}
+        run: |
+          echo "PR_HEAD=$PR_HEAD_REPO_OWNER/$PR_HEAD_REPO" >> "$GITHUB_ENV"
+      - name: Print PR information for this run
+        env:
+          PR_BASE_REF_NAME: ${{ matrix.prInfo.baseRefName }}
+          PR_HEAD_REF_NAME: ${{ matrix.prInfo.headRefName }}
+          PR_NUMBER: ${{ matrix.prInfo.number }}
+        run: |
+          echo "PR $PR_NUMBER is merging into $PR_BASE_REF_NAME from https://github.com/$PR_HEAD branch $PR_HEAD_REF_NAME."
+
+      - name: Set up JDK 9
+        uses: actions/setup-java@v1
+        with:
+          java-version: 9
+
+      - name: Set up Bazel
+        uses: abhinavsingh/setup-bazel@v3
+        with:
+          version: 4.0.0
+
+      # For reference on this & the later cache actions, see:
+      # https://github.com/actions/cache/issues/239#issuecomment-606950711 &
+      # https://github.com/actions/cache/issues/109#issuecomment-558771281. Note that these work
+      # with Bazel since Bazel can share the most recent cache from an unrelated build and still
+      # benefit from incremental build performance (assuming that actions/cache aggressively removes
+      # older caches due to the 5GB cache limit size & Bazel's large cache size).
+      - uses: actions/cache@v2
+        id: cache
+        with:
+          path: ${{ env.CACHE_DIRECTORY }}
+          key: ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-binary-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-binary-
+            ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-
+
+      # This check is needed to ensure that Bazel's unbounded cache growth doesn't result in a
+      # situation where the cache never updates (e.g. due to exceeding GitHub's cache size limit)
+      # thereby only ever using the last successful cache version. This solution will result in a
+      # few slower CI actions around the time cache is detected to be too large, but it should
+      # incrementally improve thereafter.
+      - name: Ensure cache size
+        env:
+          BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }}
+        run: |
+          # See https://stackoverflow.com/a/27485157 for reference.
+          EXPANDED_BAZEL_CACHE_PATH="${BAZEL_CACHE_DIR/#\~/$HOME}"
+          CACHE_SIZE_MB=$(du -smc $EXPANDED_BAZEL_CACHE_PATH | grep total | cut -f1)
+          echo "Total size of Bazel cache (rounded up to MBs): $CACHE_SIZE_MB"
+          # Use a 4.5GB threshold since actions/cache compresses the results, and Bazel caches seem
+          # to only increase by a few hundred megabytes across changes for unrelated branches. This
+          # is also a reasonable upper-bound (local tests as of 2021-03-31 suggest that a full build
+          # of the codebase (e.g. //...) from scratch only requires a ~2.1GB uncompressed/~900MB
+          # compressed cache).
+          if [[ "$CACHE_SIZE_MB" -gt 4500 ]]; then
+            echo "Cache exceeds cut-off; resetting it (will result in a slow build)"
+            rm -rf $EXPANDED_BAZEL_CACHE_PATH
+          fi
+
+      - name: Configure Bazel to use a local cache
+        env:
+          BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }}
+        run: |
+          EXPANDED_BAZEL_CACHE_PATH="${BAZEL_CACHE_DIR/#\~/$HOME}"
+          echo "Using $EXPANDED_BAZEL_CACHE_PATH as Bazel's cache path"
+          echo "build --disk_cache=$EXPANDED_BAZEL_CACHE_PATH" >> $HOME/.bazelrc
+        shell: bash
+
+      # This checks out the actual true develop branch separately to ensure that the stats check is
+      # run from the latest develop rather than the base branch (which might be different for
+      # chained PRs).
+      - name: Check out develop repository
+        uses: actions/checkout@v4
+        with:
+          path: develop
+
+      - name: Set up build environment
+        uses: ./develop/.github/actions/set-up-android-bazel-build-environment
+
+      - name: Check Bazel environment
+        run: |
+          cd develop
+          bazel info
+
+      - name: Check out base repository and branch
+        env:
+          PR_BASE_REF_NAME: ${{ matrix.prInfo.baseRefName }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ env.PR_BASE_REF_NAME }}
+          path: base
+
+      - name: Check out head repository and branch
+        env:
+          PR_HEAD_REF_NAME: ${{ matrix.prInfo.headRefName }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          repository: ${{ env.PR_HEAD }}
+          ref: ${{ env.PR_HEAD_REF_NAME }}
+          path: head
+
+      # Note that Bazel is shutdown between builds since multiple Bazel servers will otherwise end
+      # up being active (due to multiple repositories being used) and this can quickly overwhelm CI
+      # worker resources.
+      - name: Build Oppia dev, alpha, beta, and GA (feature branch)
+        run: |
+          cd head
+          git log -n 1
+          bazel build -- //:oppia_dev //:oppia_alpha //:oppia_beta //:oppia_ga
+          cp bazel-bin/oppia_dev.aab ../develop/oppia_dev_with_changes.aab
+          cp bazel-bin/oppia_alpha.aab ../develop/oppia_alpha_with_changes.aab
+          cp bazel-bin/oppia_beta.aab ../develop/oppia_beta_with_changes.aab
+          cp bazel-bin/oppia_ga.aab ../develop/oppia_ga_with_changes.aab
+          bazel shutdown
+
+      - name: Build Oppia dev, alpha, beta, and GA (base branch)
+        run: |
+          cd base
+          git log -n 1
+          bazel build -- //:oppia_dev //:oppia_alpha //:oppia_beta //:oppia_ga
+          cp bazel-bin/oppia_dev.aab ../develop/oppia_dev_without_changes.aab
+          cp bazel-bin/oppia_alpha.aab ../develop/oppia_alpha_without_changes.aab
+          cp bazel-bin/oppia_beta.aab ../develop/oppia_beta_without_changes.aab
+          cp bazel-bin/oppia_ga.aab ../develop/oppia_ga_without_changes.aab
+          bazel shutdown
+
+      - name: Run stats analysis tool (develop branch)
+        run: |
+          cd develop
+          git log -n 1
+          bazel run //scripts:compute_aab_differences -- \
+            $(pwd)/brief_build_summary.log $(pwd)/full_build_summary.log \
+            dev $(pwd)/oppia_dev_without_changes.aab $(pwd)/oppia_dev_with_changes.aab \
+            alpha $(pwd)/oppia_alpha_without_changes.aab $(pwd)/oppia_alpha_with_changes.aab \
+            beta $(pwd)/oppia_beta_without_changes.aab $(pwd)/oppia_beta_with_changes.aab \
+            ga $(pwd)/oppia_ga_without_changes.aab $(pwd)/oppia_ga_with_changes.aab
+
+      # Reference: https://github.com/peter-evans/create-or-update-comment#setting-the-comment-body-from-a-file.
+      # Also, for multi-line env values, see: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings.
+      - name: Extract reports for uploading & commenting
+        env:
+          PR_NUMBER: ${{ matrix.prInfo.number }}
+        id: compute-comment-body
+        run: |
+          {
+            echo 'comment_body<<EOF'
+            cat $GITHUB_WORKSPACE/develop/brief_build_summary.log
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+          FULL_BUILD_SUMMARY_FILE_NAME="full_build_summary_pr_$PR_NUMBER.log"
+          FULL_BUILD_SUMMARY_FILE_PATH="$GITHUB_WORKSPACE/develop/$FULL_BUILD_SUMMARY_FILE_NAME"
+          echo "FULL_BUILD_SUMMARY_FILE_NAME=$FULL_BUILD_SUMMARY_FILE_NAME" >> "$GITHUB_ENV"
+          echo "FULL_BUILD_SUMMARY_FILE_PATH=$FULL_BUILD_SUMMARY_FILE_PATH" >> "$GITHUB_ENV"
+          cp "$GITHUB_WORKSPACE/develop/full_build_summary.log" "$FULL_BUILD_SUMMARY_FILE_PATH"
+
+      - name: Add build stats summary comment
+        env:
+          PR_NUMBER: ${{ matrix.prInfo.number }}
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          issue-number: ${{ env.PR_NUMBER }}
+          body: ${{ steps.compute-comment-body.outputs.comment_body }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.FULL_BUILD_SUMMARY_FILE_NAME }}
+          path: ${{ env.FULL_BUILD_SUMMARY_FILE_PATH }}

WORKSPACE

@@ -4,15 +4,16 @@ This file lists and imports all external dependencies needed to build Oppia Andr
 
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar")
+load("//:build_vars.bzl", "BUILD_SDK_VERSION", "BUILD_TOOLS_VERSION")
 load("//third_party:versions.bzl", "HTTP_DEPENDENCY_VERSIONS", "MAVEN_REPOSITORIES", "get_maven_dependencies")
 
 # Android SDK configuration. For more details, see:
 # https://docs.bazel.build/versions/master/be/android.html#android_sdk_repository
 # TODO(#1542): Sync Android SDK version with the manifest.
 android_sdk_repository(
     name = "androidsdk",
-    api_level = 33,
-    build_tools_version = "29.0.2",
+    api_level = BUILD_SDK_VERSION,
+    build_tools_version = BUILD_TOOLS_VERSION,
 )
 
 # Oppia's backend proto API definitions.
@@ -160,6 +161,13 @@ git_repository(
     shallow_since = "1679426649 -0700",
 )
 
+git_repository(
+    name = "archive_patcher",
+    commit = "d1c18b0035d5f669ddaefadade49cae0748f9df2",
+    remote = "https://github.com/oppia/archive-patcher",
+    shallow_since = "1642022460 -0800",
+)
+
 bind(
     name = "databinding_annotation_processor",
     actual = "//tools/android:compiler_annotation_processor",
@@ -214,17 +222,24 @@ load("@maven//:defs.bzl", "pinned_maven_install")
 
 pinned_maven_install()
 
-http_jar(
-    name = "guava_android",
-    sha256 = HTTP_DEPENDENCY_VERSIONS["guava_android"]["sha"],
-    urls = [
-        "{0}/com/google/guava/guava/{1}-android/guava-{1}-android.jar".format(
-            url_base,
-            HTTP_DEPENDENCY_VERSIONS["guava_android"]["version"],
-        )
-        for url_base in DAGGER_REPOSITORIES + MAVEN_REPOSITORIES
-    ],
-)
+[
+    http_jar(
+        name = "guava_%s" % guava_type,
+        sha256 = HTTP_DEPENDENCY_VERSIONS["guava_%s" % guava_type]["sha"],
+        urls = [
+            "{0}/com/google/guava/guava/{1}-{2}/guava-{1}-{2}.jar".format(
+                url_base,
+                HTTP_DEPENDENCY_VERSIONS["guava_%s" % guava_type]["version"],
+                guava_type,
+            )
+            for url_base in DAGGER_REPOSITORIES + MAVEN_REPOSITORIES
+        ],
+    )
+    for guava_type in [
+        "android",
+        "jre",
+    ]
+]
 
 http_jar(
     name = "kotlinx-coroutines-core-jvm",

build_vars.bzl

@@ -0,0 +1,2 @@
+BUILD_SDK_VERSION = 33
+BUILD_TOOLS_VERSION = "29.0.2"

oppia_android_application.bzl

@@ -280,7 +280,7 @@ _bundle_module_zip_into_deployable_aab = rule(
         "_bundletool_tool": attr.label(
             executable = True,
             cfg = "host",
-            default = "//third_party:android_bundletool",
+            default = "//third_party:android_bundletool_binary",
         ),
     },
     implementation = _bundle_module_zip_into_deployable_aab_impl,
@@ -316,7 +316,7 @@ _generate_apks_and_install = rule(
         "_bundletool_tool": attr.label(
             executable = True,
             cfg = "host",
-            default = "//third_party:android_bundletool",
+            default = "//third_party:android_bundletool_binary",
         ),
     },
     executable = True,

scripts/BUILD.bazel

@@ -42,6 +42,16 @@ package_group(
     packages = ["//scripts/src/java/..."],
 )
 
+kt_jvm_binary(
+    name = "compute_aab_differences",
+    testonly = True,
+    data = ["@androidsdk//:aapt2_binary"],
+    main_class = "org.oppia.android.scripts.apkstats.ComputeAabDifferencesKt",
+    runtime_deps = [
+        "//scripts/src/java/org/oppia/android/scripts/apkstats:compute_aab_differences_lib",
+    ],
+)
+
 kt_jvm_binary(
     name = "compute_affected_tests",
     testonly = True,

scripts/src/java/org/oppia/android/scripts/apkstats/Aapt2Client.kt

@@ -0,0 +1,69 @@
+package org.oppia.android.scripts.apkstats
+
+import org.oppia.android.scripts.common.CommandExecutor
+import org.oppia.android.scripts.common.CommandExecutorImpl
+import org.oppia.android.scripts.common.ScriptBackgroundCoroutineDispatcher
+import java.io.File
+
+/**
+ * General utility for interfacing with AAPT2 in the local system at the specified working directory
+ * path and contained within the specified Android SDK (per the given path).
+ *
+ * Note that in order for binary dependencies to utilize this client, they must add a 'data'
+ * dependency on the AAPT2 binary included as part of the Android SDK, e.g.:
+ *
+ * ```bazel
+ * data = ["@androidsdk//:aapt2_binary"]
+ * ```
+ *
+ * @property workingDirectoryPath the path to the working directory in which instances of AAPT2
+ *     should be executed
+ * @property buildToolsVersion the version of Android build tools installed & that should be used.
+ *     This value should be coordinated with the build system used by the APKs accessed by this
+ *     utility.
+ * @param scriptBgDispatcher the [ScriptBackgroundCoroutineDispatcher] to be used for running the
+ *     AAPT2 command
+ * @property commandExecutor the [CommandExecutor] to use when accessing AAPT2
+ */
+class Aapt2Client(
+  private val workingDirectoryPath: String,
+  private val buildToolsVersion: String,
+  scriptBgDispatcher: ScriptBackgroundCoroutineDispatcher,
+  private val commandExecutor: CommandExecutor = CommandExecutorImpl(scriptBgDispatcher)
+) {
+  private val workingDirectory by lazy { File(workingDirectoryPath) }
+  // Note that this pathing will not work by default on Windows (since executables end with '.exe').
+  private val aapt2Path by lazy {
+    File("external/androidsdk", "build-tools/$buildToolsVersion/aapt2").absolutePath
+  }
+
+  // CLI reference: https://developer.android.com/studio/command-line/apkanalyzer.
+
+  /** Returns the permissions dump as reported by AAPT2 for the specified APK. */
+  fun dumpPermissions(inputApkPath: String): List<String> {
+    return executeApkAnalyzerCommand("dump", "permissions", inputApkPath)
+  }
+
+  /** Returns the resources dump as reported by AAPT2 for the specified APK. */
+  fun dumpResources(inputApkPath: String): List<String> {
+    return executeApkAnalyzerCommand("dump", "resources", inputApkPath)
+  }
+
+  /**
+   * Returns badging information, that is, high-level details like supported locales and densities,
+   * from the specified APK's manifest.
+   */
+  fun dumpBadging(inputApkPath: String): List<String> {
+    return executeApkAnalyzerCommand("dump", "badging", inputApkPath)
+  }
+
+  private fun executeApkAnalyzerCommand(vararg arguments: String): List<String> {
+    val result = commandExecutor.executeCommand(workingDirectory, aapt2Path, *arguments)
+    check(result.exitCode == 0) {
+      "Expected zero exit code (not ${result.exitCode}) for command: ${result.command}." +
+        "\nStandard output:\n${result.output.joinToString("\n")}" +
+        "\nError output:\n${result.errorOutput.joinToString("\n")}"
+    }
+    return result.output
+  }
+}