[ST] Add MustGather and publish logs from tests in AZP

.azure/templates/jobs/run_systemtests.yaml

@@ -10,6 +10,8 @@ jobs:
             jdk_version: '17'
     pool:
       vmImage: $(image)
+    variables:
+      test_log_dir: target/logs
     timeoutInMinutes: 30
     steps:
       - template: '../steps/prerequisites/install_java.yaml'
@@ -51,6 +53,7 @@ jobs:
           DOCKER_REGISTRY: registry.minikube
           DOCKER_ORG: strimzi
           DOCKER_TAG: latest
+          TEST_LOG_DIR: $(test_log_dir)
         displayName: 'Run systemtests - $(arch) - Bundle installation'
       - task: Maven@4
         inputs:
@@ -64,4 +67,13 @@ jobs:
           DOCKER_ORG: strimzi
           DOCKER_TAG: latest
           INSTALL_TYPE: Helm
-        displayName: 'Run systemtests - $(arch) - Helm installation'
+          TEST_LOG_DIR: $(test_log_dir)
+        displayName: 'Run systemtests - $(arch) - Helm installation'
+      - task: PublishBuildArtifacts@1
+        inputs:
+          # we cannot use just $(test_log_dir) as the path differs - when running the STs, context is in the `systemtest` directory
+          # and when we are publishing the logs, context is in root of the repository
+          pathtoPublish: systemtest/$(test_log_dir)
+          artifactName: systemtest-logs
+        displayName: 'Publish logs from failed tests'
+        condition: always()

pom.xml

@@ -207,7 +207,6 @@
                 <groupId>org.junit.jupiter</groupId>
                 <artifactId>junit-jupiter-api</artifactId>
                 <version>${junit-jupiter.version}</version>
-                <scope>test</scope>
             </dependency>
             <dependency>
                 <groupId>org.junit.jupiter</groupId>
@@ -248,6 +247,11 @@
                 <artifactId>test-frame-kubernetes</artifactId>
                 <version>${test-frame.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.skodjob</groupId>
+                <artifactId>test-frame-log-collector</artifactId>
+                <version>${test-frame.version}</version>
+            </dependency>
             <dependency>
                 <groupId>io.fabric8</groupId>
                 <artifactId>kubernetes-model-apiextensions</artifactId>

systemtest/pom.xml

@@ -59,7 +59,6 @@
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>io.strimzi.access-operator</groupId>
@@ -73,6 +72,10 @@
             <groupId>io.skodjob</groupId>
             <artifactId>test-frame-kubernetes</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.skodjob</groupId>
+            <artifactId>test-frame-log-collector</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.fabric8</groupId>
             <artifactId>kubernetes-model-apiextensions</artifactId>

systemtest/src/main/java/io/strimzi/kafka/access/Environment.java

@@ -10,6 +10,7 @@
 public class Environment {
 
     private static final TestEnvironmentVariables ENVIRONMENT_VARIABLES = new TestEnvironmentVariables();
+    public static final String USER_PATH = System.getProperty("user.dir");
 
     //---------------------------------------
     // Env variables initialization
@@ -26,6 +27,9 @@ public class Environment {
     private static final String OPERATOR_TAG_ENV = "DOCKER_TAG";
     public static final String OPERATOR_TAG = ENVIRONMENT_VARIABLES.getOrDefault(OPERATOR_TAG_ENV, null);
 
+    private static final String TEST_LOG_DIR_ENV = "TEST_LOG_DIR";
+    public static final String TEST_LOG_DIR = ENVIRONMENT_VARIABLES.getOrDefault(TEST_LOG_DIR_ENV, USER_PATH + "/../systemtest/target/logs/");
+
     static {
         ENVIRONMENT_VARIABLES.logEnvironmentVariables();
     }

systemtest/src/main/java/io/strimzi/kafka/access/TestConstants.java

@@ -19,13 +19,14 @@ public interface TestConstants {
     // Strimzi related constants
     //--------------------------
     // in case of change in the pom.xml, update this one as well please
-    String STRIMZI_API_VERSION = "0.41.0";
+    String STRIMZI_API_VERSION = "0.48.0";
 
     //--------------------------
     // Resource types
     //--------------------------
     String NAMESPACE = "Namespace";
     String DEPLOYMENT = "Deployment";
+    String SECRET = "Secret";
     String SERVICE_ACCOUNT = "ServiceAccount";
     String CLUSTER_ROLE = "ClusterRole";
     String CLUSTER_ROLE_BINDING = "ClusterRoleBinding";
@@ -56,4 +57,10 @@ public interface TestConstants {
     //--------------------------
     long GLOBAL_POLL_INTERVAL_SHORT_MS = Duration.ofSeconds(1).toMillis();
     long GLOBAL_TIMEOUT_SHORT_MS = Duration.ofMinutes(2).toMillis();
+
+    //--------------------------
+    // Test labeling
+    //--------------------------
+    String TEST_CASE_NAME_LABEL = "test.case";
+    String TEST_SUITE_NAME_LABEL = "test.suite";
 }

systemtest/src/main/java/io/strimzi/kafka/access/log/MustGatherImpl.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright Strimzi authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.strimzi.kafka.access.log;
+
+import io.fabric8.kubernetes.api.model.LabelSelectorBuilder;
+import io.skodjob.testframe.LogCollector;
+import io.skodjob.testframe.LogCollectorBuilder;
+import io.skodjob.testframe.clients.KubeClient;
+import io.skodjob.testframe.clients.cmdClient.Kubectl;
+import io.skodjob.testframe.interfaces.MustGatherSupplier;
+import io.strimzi.api.kafka.model.kafka.Kafka;
+import io.strimzi.api.kafka.model.user.KafkaUser;
+import io.strimzi.kafka.access.Environment;
+import io.strimzi.kafka.access.TestConstants;
+import io.strimzi.kafka.access.model.KafkaAccess;
+import io.strimzi.kafka.access.utils.TestUtils;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Implementation class for {@link io.skodjob.testframe.annotations.MustGather}, containing handling
+ * of the log collection in case of test failure.
+ */
+public final class MustGatherImpl implements MustGatherSupplier {
+    private final LogCollector defaultLogCollector = new LogCollectorBuilder()
+            .withKubeClient(new KubeClient())
+            .withKubeCmdClient(new Kubectl())
+            .withRootFolderPath(Environment.TEST_LOG_DIR)
+            .withNamespacedResources(
+                List.of(
+                    TestConstants.SECRET.toLowerCase(Locale.ROOT),
+                    TestConstants.DEPLOYMENT.toLowerCase(Locale.ROOT),
+                    Kafka.RESOURCE_SINGULAR,
+                    KafkaUser.RESOURCE_SINGULAR,
+                    KafkaAccess.KIND
+                ).toArray(new String[0])
+            )
+            .withCollectPreviousLogs()
+            .build();
+
+    private static final String CURRENT_DATE;
+
+    static {
+        // Get current date to create a unique folder
+        DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss");
+        dateTimeFormatter = dateTimeFormatter.withZone(ZoneId.of("GMT"));
+        CURRENT_DATE = dateTimeFormatter.format(LocalDateTime.now());
+    }
+
+    @Override
+    public void saveKubernetesState(ExtensionContext extensionContext) {
+        String testClass = extensionContext.getRequiredTestClass().getName();
+        String testClassShortName = TestUtils.removePackageName(testClass);
+        String testCase = extensionContext.getRequiredTestClass() != null ? extensionContext.getRequiredTestClass().getSimpleName() : null;
+
+        LogCollector logCollector = new LogCollectorBuilder(defaultLogCollector)
+            .withRootFolderPath(buildFullPathToLogs(testClass, testCase).toString())
+            .build();
+
+        // firstly collect resources for class
+        logCollector.collectFromNamespacesWithLabels(
+            new LabelSelectorBuilder()
+                .withMatchLabels(Map.of(TestConstants.TEST_SUITE_NAME_LABEL, testClassShortName))
+                .build()
+        );
+
+        // then collect from Namespaces in test itself
+        if (testCase != null) {
+            logCollector.collectFromNamespacesWithLabels(
+                new LabelSelectorBuilder()
+                    .withMatchLabels(
+                        Map.of(
+                            TestConstants.TEST_SUITE_NAME_LABEL, testClassShortName,
+                            TestConstants.TEST_CASE_NAME_LABEL, testCase
+                        )
+                    )
+                    .build()
+            );
+        }
+    }
+
+    /**
+     * Method that checks existence of the folder on specified path.
+     * From there, if there are no sub-dirs created - for each of the test-case run/re-run - the method returns the
+     * full path containing the specified path and index (1).
+     * Otherwise, it lists all the directories, filtering all the folders that are indexes, takes the last one, and returns
+     * the full path containing specified path and index increased by one.
+     *
+     * @param rootPathToLogsForTestCase     complete path for test-class/test-class and test-case logs
+     *
+     * @return  full path to logs directory built from specified root path and index
+     */
+    private Path checkPathAndReturnFullRootPathWithIndexFolder(Path rootPathToLogsForTestCase) {
+        File logsForTestCase = rootPathToLogsForTestCase.toFile();
+        int index = 1;
+
+        if (logsForTestCase.exists()) {
+            String[] filesInLogsDir = logsForTestCase.list();
+
+            if (filesInLogsDir != null && filesInLogsDir.length > 0) {
+                List<String> indexes = Arrays
+                    .stream(filesInLogsDir)
+                    .filter(file -> {
+                        try {
+                            Integer.parseInt(file);
+                            return true;
+                        } catch (NumberFormatException e) {
+                            return false;
+                        }
+                    })
+                    .sorted()
+                    .toList();
+
+                // check if there is actually something in the list of folders
+                if (!indexes.isEmpty()) {
+                    // take the highest index and increase it by one for a new directory
+                    index = Integer.parseInt(indexes.get(indexes.size() - 1)) + 1;
+                }
+            }
+        }
+
+        return rootPathToLogsForTestCase.resolve(String.valueOf(index));
+    }
+
+    /**
+     * Method for building the full path to logs for specified test-class and test-case.
+     *
+     * @param testClass     name of the test-class
+     * @param testCase      name of the test-case
+     *
+     * @return full path to the logs for test-class and test-case, together with index
+     */
+    private Path buildFullPathToLogs(String testClass, String testCase) {
+        Path rootPathToLogsForTestCase = Path.of(Environment.TEST_LOG_DIR, CURRENT_DATE, testClass);
+
+        if (testCase != null) {
+            rootPathToLogsForTestCase = rootPathToLogsForTestCase.resolve(testCase);
+        }
+
+        return checkPathAndReturnFullRootPathWithIndexFolder(rootPathToLogsForTestCase);
+    }
+}

systemtest/src/main/java/io/strimzi/kafka/access/utils/TestUtils.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Strimzi authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.strimzi.kafka.access.utils;
+
+public class TestUtils {
+    private TestUtils() {}
+
+    /**
+     * Method for cutting the length of the test case in case that it's too long for having it as label in the particular resource.
+     *
+     * @param testCaseName  test case name that should be trimmed
+     *
+     * @return  trimmed test case name if needed
+     */
+    public static String trimTestCaseBaseOnItsLength(String testCaseName) {
+        // because label values `must be no more than 63 characters`
+        if (testCaseName.length() > 63) {
+            // we cut to 62 characters
+            return testCaseName.substring(0, 62);
+        }
+
+        return testCaseName;
+    }
+
+    public static String removePackageName(String testClassPath) {
+        return testClassPath.replace("io.strimzi.kafka.access.", "");
+    }
+}

systemtest/src/test/java/io/strimzi/kafka/access/AbstractST.java

@@ -4,6 +4,8 @@
  */
 package io.strimzi.kafka.access;
 
+import io.fabric8.kubernetes.api.model.Namespace;
+import io.skodjob.testframe.annotations.MustGather;
 import io.skodjob.testframe.annotations.ResourceManager;
 import io.skodjob.testframe.annotations.TestVisualSeparator;
 import io.skodjob.testframe.resources.ClusterRoleBindingType;
@@ -12,15 +14,19 @@
 import io.skodjob.testframe.resources.DeploymentType;
 import io.skodjob.testframe.resources.KubeResourceManager;
 import io.skodjob.testframe.resources.NamespaceType;
+import io.skodjob.testframe.utils.KubeUtils;
 import io.strimzi.kafka.access.installation.SetupAccessOperator;
+import io.strimzi.kafka.access.log.MustGatherImpl;
 import io.strimzi.kafka.access.resources.KafkaAccessType;
+import io.strimzi.kafka.access.utils.TestUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.TestInstance;
 
 @ResourceManager
 @TestVisualSeparator
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@MustGather(config = MustGatherImpl.class)
 @SuppressWarnings("ClassDataAbstractionCoupling")
 public abstract class AbstractST {
     protected final KubeResourceManager resourceManager = KubeResourceManager.get();
@@ -38,6 +44,30 @@ public abstract class AbstractST {
             new NamespaceType(),
             new KafkaAccessType()
         );
+
+        KubeResourceManager.get().addCreateCallback(resource -> {
+            if (resource instanceof Namespace namespace) {
+                String testClass = TestUtils.removePackageName(KubeResourceManager.get().getTestContext().getRequiredTestClass().getName());
+
+                KubeUtils.labelNamespace(
+                    namespace.getMetadata().getName(),
+                    TestConstants.TEST_SUITE_NAME_LABEL,
+                    testClass
+                );
+
+                if (KubeResourceManager.get().getTestContext().getTestMethod().isPresent()) {
+                    String testCaseName = KubeResourceManager.get().getTestContext().getRequiredTestMethod().getName();
+
+                    KubeUtils.labelNamespace(
+                        namespace.getMetadata().getName(),
+                        TestConstants.TEST_CASE_NAME_LABEL,
+                        TestUtils.trimTestCaseBaseOnItsLength(testCaseName)
+                    );
+                }
+            }
+        });
+
+        KubeResourceManager.get().setStoreYamlPath(Environment.TEST_LOG_DIR);
     }
 
     @BeforeAll