IGNITE-20903 [ducktests] Support running on JDK 17 (#11059)

Author: ivandasch

Diff for: modules/ducktests/tests/certs/functions.sh

@@ -29,7 +29,7 @@ function makeRoot() {
     rm -f "${ALIAS}.pem"
 
     keytool -genkeypair -keystore "${ALIAS}.jks" -alias "${ALIAS}" -dname "${DNAME}" -ext bc:c -storepass "${PSWD}" \
-     -keypass "${PSWD}" -storetype JKS -noprompt -v
+     -keypass "${PSWD}" -storetype JKS -keyalg RSA -noprompt -v
 
     keytool -keystore "${ALIAS}.jks" -storepass "${PSWD}" -keypass "${PSWD}" -alias "${ALIAS}" -exportcert \
      -rfc -file "${ALIAS}.pem" -v
@@ -50,7 +50,7 @@ function makeCA() {
     rm -f "${ALIAS}.pem"
 
     keytool -genkeypair -keystore "${ALIAS}.jks" -alias "${ALIAS}" -dname "${DNAME}" -ext bc:c -storepass "${PSWD}" \
-     -keypass "${PSWD}" -storetype JKS -noprompt -v
+     -keypass "${PSWD}" -storetype JKS -keyalg RSA -noprompt -v
 
     keytool -storepass "${PSWD}" -keypass "${PSWD}" -keystore "${ALIAS}.jks" -certreq -alias "${ALIAS}" \
       | keytool -storepass "${PSWD}" -keypass "${PSWD}" -keystore "${ROOT}.jks" -gencert -alias "${ROOT}" \

Diff for: modules/ducktests/tests/docker/Dockerfile

@@ -49,7 +49,7 @@ ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/"
 ARG APACHE_ARCHIVE="https://archive.apache.org/dist/"
 
 # Install binary test dependencies.
-RUN for v in "2.7.6" "2.8.0" "2.8.1" "2.9.0" "2.9.1" "2.10.0" "2.11.0" "2.11.1" "2.12.0" "2.13.0" "2.14.0"; \
+RUN for v in "2.7.6" "2.15.0"; \
     do cd /opt; \
     curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\
     unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v;\
@@ -67,14 +67,6 @@ RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELE
  && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT
 RUN mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME
 
-# Install spark
-ARG SPARK_VERSION="2.3.4"
-ARG SPARK_NAME="spark-$SPARK_VERSION"
-ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7"
-
-RUN cd /opt && curl -O $APACHE_ARCHIVE/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz
-RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME
-
 # The version of Kibosh to use for testing.
 # If you update this, also update vagrant/base.sh
 ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a"
@@ -94,7 +86,7 @@ RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMX
 RUN useradd -ms /bin/bash ducker \
     && mkdir -p /home/ducker/ \
     && rsync -aiq /root/.ssh/ /home/ducker/.ssh \
-    && chown -R ducker /home/ducker/ /mnt/ /var/log/ \
+    && chown -R ducker /home/ducker/ /mnt/ /var/log/ /opt \
     && echo "LANG=C.UTF-8" >> /home/ducker/.ssh/environment \
     && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment \
     && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile \

Diff for: modules/ducktests/tests/ignitetest/services/spark.py

@@ -38,13 +38,13 @@
 from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue
 from ignitetest.services.utils.ignite_spec import resolve_spec, SHARED_PREPARED_FILE
 from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient
-from ignitetest.services.utils.jvm_utils import JvmProcessMixin
+from ignitetest.services.utils.jvm_utils import JvmProcessMixin, JvmVersionMixin
 from ignitetest.services.utils.log_utils import monitor_log
 from ignitetest.services.utils.path import IgnitePathAware
 from ignitetest.utils.enum import constructible
 
 
-class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMixin, metaclass=ABCMeta):
+class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMixin, JvmVersionMixin, metaclass=ABCMeta):
     """
     The base class to build services aware of Ignite.
     """

Diff for: modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py

@@ -21,6 +21,7 @@
 import re
 
 from ignitetest.services.utils.decorators import memoize
+from ignitetest.services.utils.jvm_utils import java_version, java_major_version
 
 
 def ignite_jmx_mixin(node, service):
@@ -60,13 +61,16 @@ def __init__(self, node):
         self.node = node
         self.install_root = node.install_root
         self.pid = node.pids[0]
+        self.java_major = java_major_version(java_version(self.node))
 
     @property
     def jmx_util_cmd(self):
         """
         :return: jmxterm prepared command line invocation.
         """
-        return os.path.join(f"java -jar {self.install_root}/jmxterm.jar -v silent -n")
+        extra_flag = "--add-exports jdk.jconsole/sun.tools.jconsole=ALL-UNNAMED" if self.java_major >= 15 else ""
+
+        return os.path.join(f"java {extra_flag} -jar {self.install_root}/jmxterm.jar -v silent -n")
 
     @memoize
     def find_mbean(self, pattern, negative_pattern=None, domain='org.apache'):

Diff for: modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py

@@ -16,6 +16,9 @@
 """
 This module contains JVM utilities.
 """
+
+from ignitetest.services.utils.decorators import memoize
+
 DEFAULT_HEAP = "768M"
 
 JVM_PARAMS_GC_G1 = "-XX:+UseG1GC -XX:MaxGCPauseMillis=100 " \
@@ -74,6 +77,31 @@ def merge_jvm_settings(src_settings, additionals):
     return listed
 
 
+def java_major_version(version):
+    """
+    :param version: Full java version
+    :return: Java major version
+    """
+    if version:
+        version = version.split('.')
+
+        return int(version[1]) if version[0] == '1' else int(version[0])
+
+    return -1
+
+
+def java_version(node):
+    """
+    :param node: Ducktape cluster node
+    :return: java version
+    """
+    cmd = r"java -version 2>&1 | awk -F[\"\-] '/version/ {print $2}'"
+
+    raw_version = list(node.account.ssh_capture(cmd, allow_fail=False))
+
+    return raw_version[0].strip() if raw_version else ''
+
+
 def _to_map(params):
     """"""
     assert isinstance(params, (str, list)), "JVM params an be string or list only."
@@ -107,6 +135,7 @@ class JvmProcessMixin:
     """
     Mixin to work with JVM processes
     """
+
     @staticmethod
     def pids(node, java_class):
         """
@@ -118,3 +147,15 @@ def pids(node, java_class):
         cmd = "ps -C java -wwo pid,args | grep '%s' | awk -F' ' '{print $1}'" % java_class
 
         return [int(pid) for pid in node.account.ssh_capture(cmd, allow_fail=True)]
+
+
+class JvmVersionMixin:
+    """
+    Mixin to get java version on node.
+    """
+    @memoize
+    def java_version(self):
+        """
+        :return: Full java version of service.
+        """
+        return java_version(self.nodes[0])

Diff for: modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py

@@ -22,6 +22,7 @@
 from ignitetest.services.utils.control_utility import ControlUtility
 from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration
 from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration
+from ignitetest.services.utils.jvm_utils import java_major_version
 from ignitetest.services.utils.ssl.ssl_params import is_ssl_enabled
 from ignitetest.utils import cluster, ignite_versions, ignore_if
 from ignitetest.utils.ignite_test import IgniteTest
@@ -52,6 +53,10 @@ def upgrade_test(self, versions, ignite_version):
                             "DataLoaderAndCheckerApplication"
         )
 
+        java_version = service.java_version()
+        if java_major_version(java_version) > 11:
+            return f"Skipped on java {java_version}"
+
         for version in versions:
             service.config = IgniteConfiguration(
                 data_storage=DataStorageConfiguration(default=DataRegionConfiguration(persistence_enabled=True)),

Diff for: modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py

@@ -19,7 +19,6 @@
 
 from ignitetest.services.ignite import IgniteService
 from ignitetest.services.ignite_app import IgniteApplicationService
-from ignitetest.services.spark import SparkService
 from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster
 from ignitetest.services.utils.ignite_configuration import IgniteConfiguration
 from ignitetest.services.zk.zookeeper import ZookeeperService
@@ -67,15 +66,6 @@ def test_ignite_app_start_stop(self, ignite_version):
         app.stop()
         ignite.stop()
 
-    @cluster(num_nodes=2)
-    def test_spark_start_stop(self):
-        """
-        Test that SparkService correctly start and stop
-        """
-        spark = SparkService(self.test_context, num_nodes=2)
-        spark.start()
-        spark.stop()
-
     @cluster(num_nodes=3)
     def test_zk_start_stop(self):
         """

Diff for: modules/ducktests/tests/ignitetest/tests/smoke_test.py

@@ -108,7 +108,11 @@ def __repr__(self):
 V_2_14_0 = IgniteVersion("2.14.0")
 LATEST_2_14 = V_2_14_0
 
+# 2.15.x versions
+V_2_15_0 = IgniteVersion("2.15.0")
+LATEST_2_15 = V_2_15_0
+
 # if you updated the LATEST version
 # please check DEV version in 'tests/ignitetest/__init__.py'
-LATEST = LATEST_2_14
+LATEST = LATEST_2_15
 OLDEST = V_2_7_6