Merge pull request #2697 from newrelic/FlakyUnitTests

Better handle flaky unit tests

Author: jbedell-newrelic

.github/workflows/GHA-Unit-Tests.yaml

@@ -18,14 +18,15 @@ on:
 
 jobs:
   unit-test:
-    name: Java ${{ matrix.java-version }}
+    name: Java ${{ matrix.java-version }} - ${{ matrix.group }}
     timeout-minutes: 150
     runs-on: ubuntu-24.04
     strategy:
       # max-parallel: 1 ## used to force sequential vs. concurrent
       fail-fast: false
       matrix:
         java-version: [ 8, 11, 17, 21, 25 ]
+        group: [flaky, nonForkedTests, forkedTests]
     steps:
       - name: Checkout Java agent
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
@@ -43,23 +44,23 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment
 
-      - name: Run unit tests that do not require a forked JVM (attempt 1)
+      - name: Run unit tests (attempt 1)
         id: run_tests_1
         continue-on-error: true
         timeout-minutes: 45
-        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -PnonForkedTests --continue
+        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -P${{ matrix.group }} --continue
 
-      - name: Run unit tests that do not require a forked JVM (attempt 2)
+      - name: Run unit tests (attempt 2)
         id: run_tests_2
         continue-on-error: true
         timeout-minutes: 45
         if: steps.run_tests_1.outcome == 'failure'
-        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -PnonForkedTests --continue
+        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -P${{ matrix.group }} --continue
 
-      - name: Run unit tests that do not require a forked JVM (attempt 3)
+      - name: Run unit tests (attempt 3)
         timeout-minutes: 45
         if: steps.run_tests_2.outcome == 'failure'
-        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -PnonForkedTests --continue
+        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -P${{ matrix.group }} --continue
 
       - name: Upload coverage to Codecov
         if: matrix.java-version == '17'
@@ -73,63 +74,16 @@ jobs:
         if: matrix.java-version == '17'
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4
         with:
-          name: non-forked-jacoco-reports-java-${{ matrix.java-version }}
+          name: ${{ matrix.group }}-jacoco-reports-java-${{ matrix.java-version }}
           path: |
             **/build/reports/jacoco/**
 
-      - name: Capture build reports (non forked)
+      - name: Capture build reports
         # If previous step fails, run this step regardless
         if: failure()
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4
         with:
-          name: non-forked-tests-results-java-${{ matrix.java-version }}
-          # The regex for the path below will capture unit test HTML reports generated by gradle for all
-          # related modules: (agent-bridge, newrelic-java, newrelic-api, etc).
-          # However, it's critical that the previous build step does a ./gradlew clean or the regex will capture test reports
-          # that were leftover in unrelated modules for functional and instrumentation tests.
-          path: |
-            **/build/reports/tests/*
-
-      - name: Run unit tests that require a forked JVM (attempt 1)
-        id: run_forked_tests_1
-        continue-on-error: true
-        timeout-minutes: 45
-        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -PforkedTests --continue
-
-      - name: Run unit tests that require a forked JVM (attempt 2)
-        id: run_forked_tests_2
-        continue-on-error: true
-        timeout-minutes: 45
-        if: steps.run_forked_tests_1.outcome == 'failure'
-        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -PforkedTests --continue
-
-      - name: Run unit tests that require a forked JVM (attempt 3)
-        id: run_forked_tests_3
-        timeout-minutes: 45
-        if: steps.run_forked_tests_2.outcome == 'failure'
-        run: ./gradlew $GRADLE_OPTIONS test -x :functional_test:test -x :newrelic-scala3-api:test -x :newrelic-scala-api:test -x :newrelic-scala-cats-api:test -x :newrelic-cats-effect3-api:test -x :newrelic-scala-monix-api:test -x :newrelic-scala-zio-api:test -x :newrelic-scala-zio2-api:test -Ptest${{ matrix.java-version }} -PnoInstrumentation -PforkedTests --continue
-
-      - name: Upload coverage to Codecov
-        if: matrix.java-version == '17'
-        uses: codecov/codecov-action@6d798873df2b1b8e5846dba6fb86631229fbcb17 # pin@v4
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: '**/build/reports/jacoco/test/jacocoTestReport.xml'
-          fail_ci_if_error: false #default is false, but being explicit about what to expect.
-
-      - name: Capture Jacoco reports
-        if: matrix.java-version == '17'
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4
-        with:
-          name: forked-jacoco-reports-java-${{ matrix.java-version }}
-          path: |
-            **/build/reports/jacoco/**
-
-      - name: Capture build reports (forked)
-        if: failure()
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4
-        with:
-          name: forked-tests-results-java-${{ matrix.java-version }}
+          name: ${{ matrix.group }}-tests-results-java-${{ matrix.java-version }}
           # The regex for the path below will capture unit test HTML reports generated by gradle for all
           # related modules: (agent-bridge, newrelic-java, newrelic-api, etc).
           # However, it's critical that the previous build step does a ./gradlew clean or the regex will capture test reports
@@ -150,12 +104,12 @@ jobs:
           git config --global user.name "GitHub Actions Bot"
           git config --global user.email "[email protected]"
 
-      - name: Download test reports Java ${{ matrix.java-version }}
+      - name: Download test reports Java ${{ matrix.java-version }} - ${{ matrix.group }}
         if: ${{ failure() }}
         uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # pin@v4
         with:
-          name: unit-tests-results-java-${{ matrix.java-version }}
-          path: gh-pages/reports/${{ inputs.agent-ref }}/${{ github.run_id }}/unit-tests-results-java-${{ matrix.java-version }}
+          name: unit-tests-results-java-${{ matrix.java-version }}-${{ matrix.group }}
+          path: gh-pages/reports/${{ inputs.agent-ref }}/${{ github.run_id }}/unit-tests-results-java-${{ matrix.java-version }}-${{ matrix.group }}
 
       - name: Set up Python
         if: ${{ failure() }}
@@ -168,7 +122,7 @@ jobs:
         if: ${{ failure() }}
         run: |
           pip install beautifulsoup4
-          python gh-pages/utils/writeTestsFailuresPage.py gh-pages/reports/${{ inputs.agent-ref }}/${{ github.run_id }}/unit-tests-results-java-${{ matrix.java-version }}
+          python gh-pages/utils/writeTestsFailuresPage.py gh-pages/reports/${{ inputs.agent-ref }}/${{ github.run_id }}/unit-tests-results-java-${{ matrix.java-version }}-${{ matrix.group }}
 
       - name: Commit and push test reports to gh-pages
         if: ${{ failure() }}

newrelic-agent/build.gradle

@@ -326,11 +326,13 @@ test {
     forkEvery = 1
     maxParallelForks = Runtime.runtime.availableProcessors()
     useJUnit {
-        if (project.hasProperty("forkedTests")) {
+        if (project.hasProperty("flaky")) {
+            includeCategories("com.newrelic.test.marker.Flaky")
+        } else if (project.hasProperty("forkedTests")) {
             includeCategories("com.newrelic.test.marker.RequiresFork")
         } else if (project.hasProperty("nonForkedTests")) {
             forkEvery = 10
-            excludeCategories("com.newrelic.test.marker.RequiresFork")
+            excludeCategories("com.newrelic.test.marker.RequiresFork", "com.newrelic.test.marker.Flaky")
         }
     }
 

newrelic-agent/src/test/java/com/newrelic/agent/jfr/JfrServiceTest.java

@@ -13,6 +13,7 @@
 import com.newrelic.jfr.ThreadNameNormalizer;
 import com.newrelic.jfr.daemon.DaemonConfig;
 import com.newrelic.jfr.daemon.JfrRecorderException;
+import com.newrelic.test.marker.Flaky;
 import com.newrelic.test.marker.IBMJ9IncompatibleTest;
 import org.junit.Before;
 import org.junit.Test;
@@ -126,8 +127,9 @@ public void jfrLoopDoesNotStartWhenIsEnabledIsTrueAndHighSecurityIsTrue() throws
         verify(spyJfr, times(0)).startJfrLoop();
     }
 
-    @Category( IBMJ9IncompatibleTest.class )
+    @Category( { IBMJ9IncompatibleTest.class, Flaky.class } )
     @Test
+    // Flaky note: org.mockito.exceptions.verification.WantedButNotInvoked on verify(spyJfr, timeout(100)).startJfrLoop();
     public void jfrLoopDoesStart() {
         JfrService jfrService = new JfrService(jfrConfig, agentConfig);
         JfrService spyJfr = spy(jfrService);

newrelic-agent/src/test/java/com/newrelic/agent/service/module/JarCollectorServiceProcessorTest.java

@@ -11,7 +11,9 @@
 import com.newrelic.agent.config.AgentConfig;
 import com.newrelic.agent.config.JarCollectorConfig;
 import com.newrelic.api.agent.Logger;
+import com.newrelic.test.marker.Flaky;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.mockito.ArgumentMatchers;
 
 import javax.servlet.jsp.JspPage;
@@ -94,6 +96,11 @@ private AgentConfig getMockConfig() {
     }
 
     @Test
+    @Category( Flaky.class )
+    // Flaky note: Somehow the elapsedMillis is sometimes < 4000, which probably shouldn't happen, but
+    // I haven't investigated the warmup time of the SmoothBursty RateLimiter class
+    //  Also, I put this in a loop and ran locally 1000 times and never saw a failure, so it
+    // may be another strange timing issue in GH.
     public void applyWithRateLimit() throws URISyntaxException {
         AgentConfig config = getMockConfig();
         when(config.getJarCollectorConfig().getJarsPerSecond()).thenReturn(10);

newrelic-agent/src/test/java/com/newrelic/agent/tracing/samplers/AdaptiveSamplerTest.java

@@ -2,9 +2,11 @@
 
 import com.newrelic.agent.MockServiceManager;
 import com.newrelic.agent.tracing.DistributedTraceUtil;
+import com.newrelic.test.marker.Flaky;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 import java.util.List;
 import java.util.ArrayList;
@@ -170,6 +172,8 @@ public void testExponentialBackoff() throws InterruptedException, ExecutionExcep
     }
 
     @Test
+    @Category( Flaky.class )
+    // Flaky note: expectedSampled ends up being 15 and 16s and 17s are fairly common
     public void testCalculatePriorityMultithreaded() throws InterruptedException {
         int target = 10;
         int reportPeriod = 5;
@@ -243,7 +247,7 @@ public int runSamplerConcurrentAndGetSampled(AdaptiveSampler sampler, long total
         AtomicInteger totalSampled = new AtomicInteger(0);
         //in this example, the wait is random, and the load is distributed across 4 threads.
         Random random = new Random();
-        int MAX_WAIT_BETWEEN_SAMPLES = 500;
+        int MAX_WAIT_BETWEEN_SAMPLES = 50;
         //Start it up
         long startTime = System.currentTimeMillis();
         for (int i = 0; i < nThreads ; i++) {

newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java

@@ -31,12 +31,14 @@
 import com.newrelic.agent.stats.RecordDataUsageMetric;
 import com.newrelic.agent.stats.StatsImpl;
 import com.newrelic.agent.stats.StatsService;
+import com.newrelic.test.marker.Flaky;
 import org.hamcrest.CoreMatchers;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mock;
@@ -243,6 +245,9 @@ public void dataReceived(String method, String encoding, String uri, Map<?, ?> r
     }
 
     @Test
+    @Category( Flaky.class )
+    // Flaky note: java.lang.ArrayIndexOutOfBoundsException inside assertDataUsageMetricValues
+    // on Object rawArgument = invocation.getRawArguments()[0];
     public void testDataUsageSupportability() throws Exception {
         AgentConfig config = AgentConfigImpl.createAgentConfig(configMap());
         HttpClientWrapper wrapperEmptyReturn = getHttpClientWrapper(ReadResult.create(

newrelic-agent/src/test/java/com/newrelic/bootstrap/EmbeddedJarFilesImplTest.java

@@ -1,11 +1,13 @@
 package com.newrelic.bootstrap;
 
+import com.newrelic.test.marker.Flaky;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
@@ -102,6 +104,9 @@ public void cleanupStaleTempJarFiles_onlyDeletesJarFiles() throws Exception {
     }
 
     @Test
+    @Category( Flaky.class )
+    // Flaky note: GHA can sometimes take a very long time to go from afterThresholdJar.setLastModified to
+    // setting the cutoff time in the Impl class, which leads to the file being erroneously deleted
     public void cleanupStaleTempJarFiles_respectsThresholdExactly() throws Exception {
         long now = System.currentTimeMillis();
         long fiveHoursInMillis = 5 * 60 * 60 * 1000L;
@@ -112,7 +117,7 @@ public void cleanupStaleTempJarFiles_respectsThresholdExactly() throws Exception
 
         // Create jar just after threshold - should not be deleted
         File afterThresholdJar = createTempJarFile(BootstrapLoader.AGENT_BRIDGE_JAR_NAME + "33333.jar", 0);
-        afterThresholdJar.setLastModified(now - fiveHoursInMillis + 1);
+        afterThresholdJar.setLastModified(now - fiveHoursInMillis + 15000); // large padding for GHA warmup
 
         setThresholdAndCleanup(5);
 

test-annotations/src/main/java/com/newrelic/test/marker/Flaky.java

@@ -0,0 +1,32 @@
+/*
+ *
+ *  * Copyright 2026 New Relic Corporation. All rights reserved.
+ *  * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package com.newrelic.test.marker;
+
+/**
+ * <p>
+ * Marker interface to denote a unit/functional/instrumentation test has proven flaky.
+ * This allows us to categorize these and more easily re-run them without having to
+ * re-run the entire test suite.
+ * </p>
+ * <p>
+ * This category currently only affects tests in newrelic-agent when the tests are run with
+ * <code>-Pflaky</code>.
+ * </p>
+ * <p>
+ * <b>Note: This annotation will override RequiresFork.</b>
+ * </p>
+ *
+ * <p>
+ * To mark your test with this category, add the following annotation to the class.
+ * <pre>
+ *  {@code @Category({ Flaky.class })}
+ * </pre>
+ * </p>
+ */
+public interface Flaky {
+}