spike: Run the guide tests in groups under a forked executor (#1428)

* Run the guide tests in a forked executor

We seem to have issues with guides hanging, or memory exhaustion.

This PR runs each guide in a forked process to hopefully isolate the tests and improve this situation

* Move aws env vars from runner to metadata

* Env vars required for xray test

* Set max heap for forked jvm

* Log all current docker containers to see if they're using up memory

* Fix script

* Up to gradle 8.5, limit workers to 1 and add Xmx setting

* Split guide tests into 4 groups

* Fix groups

* Env for dynamo

* Fetch the number of groups from the build

* Fix matrix

* Fix again

* Fix again

* And fix again

* Try adding cache

* Test with 5 groups

* Remove docker debug, move group count out of buildSrc

Author: timyates

.github/workflows/gradle.yml

@@ -9,33 +9,61 @@ on:
       - master
       - '[1-9]+.[0-9]+.x'
 jobs:
+  test_matrix:
+    name: Generate Guide Test Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.test_matrix_step.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'adopt'
+          java-version: '17'
+      - uses: actions/cache@v3
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Generate Guide Test Matrix
+        id: test_matrix_step
+        run: |
+          echo "Searching for groups..."
+          gradleTasks=$(./gradlew tasks | grep "Run group of guide tests" | sed 's/\(.*\) - Run group of guide tests/\"\1\"/' | tr '\n' ',' | sed 's/.$//')
+          tasksEntry="\"group_test_tasks\":[$(echo $gradleTasks)]"
+          matrixValue="{$(echo $tasksEntry)}"
+          echo "Created matrix: $matrixValue"
+          echo "matrix=$matrixValue" >> $GITHUB_OUTPUT
   alltests:
+    name: Running ${{ matrix.group }} with Java ${{ matrix.java }}
+    needs: test_matrix
     runs-on: ubuntu-latest
     strategy:
       matrix:
         java: ["17", "21"]
+        group: ${{ fromJson(needs.test_matrix.outputs.matrix).group_test_tasks }}
     env:
       JDK_VERSION: ${{ matrix.java }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.gradle/caches
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
       - name: Set up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
+          distribution: 'adopt'
           java-version: ${{ matrix.java }}
       - name: Run All Guide Tests
-        run: './gradlew runAllGuideTests'
+        run: './gradlew ${{ matrix.group }}'
         env:
-          AWS_ACCESS_KEY_ID: XXX
-          AWS_SECRET_ACCESS_KEY: YYY
-          AWS_REGION: us-east-1
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY }}
           GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GRADLE_ENTERPRISE_CACHE_USERNAME }}
           GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GRADLE_ENTERPRISE_CACHE_PASSWORD }}

buildSrc/src/main/groovy/io/micronaut/guides/GuideMetadata.groovy

@@ -36,6 +36,8 @@ class GuideMetadata {
 
     List<String> zipIncludes
 
+    Map<String, String> env
+
     List<App> apps
 
     List<String> getTags() {

buildSrc/src/main/groovy/io/micronaut/guides/GuideProjectGenerator.groovy

@@ -1,6 +1,5 @@
 package io.micronaut.guides
 
-import groovy.io.FileType
 import groovy.json.JsonSlurper
 import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
@@ -115,6 +114,7 @@ class GuideProjectGenerator implements AutoCloseable {
                 minimumJavaVersion: config.minimumJavaVersion,
                 maximumJavaVersion: config.maximumJavaVersion,
                 zipIncludes: config.zipIncludes ?: [],
+                env: config.env ?: [:],
                 apps: config.apps.collect { it ->
                     new App(
                             validateLicense: it.validateLicense == null ? true : it.validateLicense,
@@ -128,7 +128,8 @@ class GuideProjectGenerator implements AutoCloseable {
                             groovyFeatures: it.groovyFeatures ?: [],
                             applicationType: it.applicationType ? ApplicationType.valueOf(it.applicationType.toUpperCase()) : ApplicationType.DEFAULT,
                             excludeSource: it.excludeSource,
-                            excludeTest: it.excludeTest)
+                            excludeTest: it.excludeTest,
+                    )
                 }
         ))
     }
@@ -405,6 +406,7 @@ class GuideProjectGenerator implements AutoCloseable {
         merged.minimumJavaVersion = metadata.minimumJavaVersion ?: base.minimumJavaVersion
         merged.maximumJavaVersion = metadata.maximumJavaVersion ?: base.maximumJavaVersion
         merged.zipIncludes = metadata.zipIncludes // TODO support merging from base
+        merged.env = metadata.env ?: base.env
         merged.apps = mergeApps(base, metadata)
 
         merged

buildSrc/src/main/groovy/io/micronaut/guides/GuidesPlugin.groovy

@@ -21,6 +21,7 @@ import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.bundling.Zip
 
+import java.math.RoundingMode
 import java.nio.file.Files
 import java.nio.file.Paths
 import java.util.function.Predicate
@@ -51,6 +52,11 @@ class GuidesPlugin implements Plugin<Project> {
         GuideProjectGenerator projectGenerator = new GuideProjectGenerator()
         Directory guidesDir = project.layout.projectDirectory.dir("guides")
         Provider<Directory> codeDir = project.layout.buildDirectory.dir("code")
+        Properties testProps = guidesDir.file("tests.properties").asFile.withInputStream { inputStream ->
+            new Properties().tap {
+                load(inputStream)
+            }
+        } as Properties
         List<GuideMetadata> metadatas = GuideProjectGenerator.parseGuidesMetadata(
                 guidesDir.asFile,
                 project.extensions.extraProperties.get("metadataConfigName").toString())
@@ -101,6 +107,17 @@ class GuidesPlugin implements Plugin<Project> {
             it.description = 'Generates guide applications at build/code'
         }
 
+        int groupSize = (sampleTasks.size() / Integer.parseInt(testProps.get("numberOfTestGroups") as String))
+                .setScale(0, RoundingMode.UP).toInteger()
+
+        sampleTasks.collate(groupSize, true).eachWithIndex { List<Map<String, TaskProvider<Task>>> tasks, int i ->
+            project.tasks.register("testsGroup${i + 1}") { Task it ->
+                it.group = 'guides'
+                it.description = "Run group of guide tests"
+                it.dependsOn(tasks.collect { it[TEST_RUNNER] })
+            }
+        }
+
         project.tasks.register("runAllGuideTests") { Task it ->
             it.group = 'guides'
             it.description = 'Runs all Guide test scripts'
@@ -246,6 +263,8 @@ class GuidesPlugin implements Plugin<Project> {
             it.group = "guides $metadata.slug"
             it.description = "Run the tests for all projects generated by $metadata.slug"
 
+            it.environment.set(metadata.env)
+
             it.testScript.set(testScriptTask.flatMap { t -> t.scriptFile })
             it.guideSourceDirectory.set(project.layout.projectDirectory.dir("guides/${metadata.slug}"))
 

buildSrc/src/main/groovy/io/micronaut/guides/tasks/TestScriptRunnerTask.groovy

@@ -4,7 +4,9 @@ import groovy.transform.CompileStatic
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.MapProperty
 import org.gradle.api.tasks.CacheableTask
+import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.InputFile
 import org.gradle.api.tasks.OutputFile
@@ -29,6 +31,9 @@ abstract class TestScriptRunnerTask extends DefaultTask {
     @PathSensitive(RELATIVE)
     abstract DirectoryProperty getGuideSourceDirectory()
 
+    @Input
+    abstract MapProperty<String, String> getEnvironment()
+
     @OutputFile
     abstract RegularFileProperty getOutputFile()
 
@@ -37,7 +42,15 @@ abstract class TestScriptRunnerTask extends DefaultTask {
 
     @TaskAction
     void runScript() {
-        WorkQueue queue = workerExecutor.noIsolation()
+        Map<String, String> buildEnvironment = environment.get() + [ JAVA_HOME: System.getProperty("java.home") ]
+        logger.lifecycle("Running test script ${testScript.get()} with environment $buildEnvironment")
+
+        WorkQueue queue = workerExecutor.processIsolation(workerSpec -> {
+            workerSpec.forkOptions(options -> {
+                options.environment(buildEnvironment)
+                options.maxHeapSize = "5G"
+            });
+        })
 
         queue.submit(TestScriptRunnerWorkAction) { parameters ->
             parameters.testScript.set(testScript)

gradle.properties

@@ -2,4 +2,5 @@ org.gradle.parallel=true
 org.gradle.caching=true
 
 # Limit the max workers to prevent resource exhaustion
-org.gradle.workers.max=2
+org.gradle.workers.max=1
+org.gradle.jvmargs=-Xmx4G

gradle/wrapper/gradle-wrapper.jar

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradle/wrapper/gradle-wrapper.properties

@@ -8,6 +8,7 @@
   "languages": ["java"],
   "testFramework": "junit",
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "applicationType": "function",

guides/micronaut-aws-lambda-eventbridge-event/metadata.json

@@ -8,6 +8,7 @@
   "languages": ["java"],
   "testFramework": "junit",
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "applicationType": "function",

guides/micronaut-aws-lambda-s3-event/metadata.json

@@ -6,6 +6,7 @@
   "categories": ["AWS", "Distributed Configuration"],
   "publicationDate": "2018-10-03",
   "languages": ["java"],
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/micronaut-aws-parameter-store/metadata.json

@@ -7,6 +7,7 @@
   "publicationDate": "2021-08-10",
   "languages": ["java"],
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "applicationType": "function",

guides/micronaut-aws-secretsmanager-rotation/metadata.json

@@ -5,6 +5,7 @@
   "tags": ["aws", "secrets"],
   "categories": ["AWS", "Secrets Manager"],
   "publicationDate": "2019-05-15",
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/micronaut-aws-secretsmanager/metadata.json

@@ -5,5 +5,6 @@
   "authors": ["Álvaro Sánchez-Mariscal"],
   "tags": ["aws"],
   "categories": ["AWS"],
-  "publicationDate": "2022-03-14"
+  "publicationDate": "2022-03-14",
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" }
 }

guides/micronaut-cloud-database-aws/metadata.json

@@ -5,6 +5,7 @@
   "categories": ["Data Access"],
   "publicationDate": "2022-07-05",
   "languages": ["java"],
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/micronaut-dynamodb/metadata.json

@@ -6,6 +6,7 @@
   "categories": ["Email"],
   "base": "micronaut-email",
   "publicationDate": "2022-02-05",
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/micronaut-email-amazon-ses/metadata.json

@@ -6,6 +6,7 @@
   "tags": ["aws", "metrics"],
   "categories": ["Metrics", "AWS"],
   "publicationDate": "2022-10-10",
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [{
     "name": "default",
     "features": ["micrometer-cloudwatch"]

guides/micronaut-metrics-aws/metadata.json

@@ -6,6 +6,7 @@
   "categories": ["Distributed Tracing"],
   "publicationDate": "2022-08-04",
   "languages": ["java"],
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "bookcatalogue",

guides/micronaut-microservices-distributed-tracing-xray/metadata.json

@@ -7,6 +7,7 @@
   "categories": ["AWS", "Object Storage"],
   "publicationDate": "2022-09-20",
   "languages": ["java"],
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/micronaut-object-storage-aws/metadata.json

@@ -6,6 +6,7 @@
   "tags": ["lambda", "aws"],
   "categories": ["AWS Lambda", "GraalVM"],
   "publicationDate": "2020-08-29",
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/mn-application-aws-lambda-graalvm/metadata.json

@@ -6,6 +6,7 @@
   "categories": ["AWS Lambda"],
   "publicationDate": "2020-08-29",
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "name": "default",

guides/mn-application-aws-lambda-java-runtime/metadata.json

@@ -6,6 +6,7 @@
   "categories": ["AWS Lambda"],
   "publicationDate": "2023-10-16",
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "applicationType": "function",

guides/mn-serverless-function-aws-lambda-function-url/metadata.json

@@ -7,6 +7,7 @@
   "languages": ["java", "kotlin"],
   "publicationDate": "2020-08-29",
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "applicationType": "function",

guides/mn-serverless-function-aws-lambda-graalvm/metadata.json

@@ -6,6 +6,7 @@
   "categories": ["AWS Lambda"],
   "publicationDate": "2020-08-29",
   "maximumJavaVersion": 17,
+  "env": { "AWS_ACCESS_KEY_ID": "XXX", "AWS_SECRET_ACCESS_KEY": "YYY", "AWS_REGION": "us-east-1" },
   "apps": [
     {
       "applicationType": "function",

guides/mn-serverless-function-aws-lambda/metadata.json

@@ -0,0 +1 @@
+numberOfTestGroups=4