diff --git a/modules/ducktests/tests/certs/functions.sh b/modules/ducktests/tests/certs/functions.sh index 9a6235a85d14a..0caef590632f5 100644 --- a/modules/ducktests/tests/certs/functions.sh +++ b/modules/ducktests/tests/certs/functions.sh @@ -29,7 +29,7 @@ function makeRoot() { rm -f "${ALIAS}.pem" keytool -genkeypair -keystore "${ALIAS}.jks" -alias "${ALIAS}" -dname "${DNAME}" -ext bc:c -storepass "${PSWD}" \ - -keypass "${PSWD}" -storetype JKS -noprompt -v + -keypass "${PSWD}" -storetype JKS -keyalg RSA -noprompt -v keytool -keystore "${ALIAS}.jks" -storepass "${PSWD}" -keypass "${PSWD}" -alias "${ALIAS}" -exportcert \ -rfc -file "${ALIAS}.pem" -v @@ -50,7 +50,7 @@ function makeCA() { rm -f "${ALIAS}.pem" keytool -genkeypair -keystore "${ALIAS}.jks" -alias "${ALIAS}" -dname "${DNAME}" -ext bc:c -storepass "${PSWD}" \ - -keypass "${PSWD}" -storetype JKS -noprompt -v + -keypass "${PSWD}" -storetype JKS -keyalg RSA -noprompt -v keytool -storepass "${PSWD}" -keypass "${PSWD}" -keystore "${ALIAS}.jks" -certreq -alias "${ALIAS}" \ | keytool -storepass "${PSWD}" -keypass "${PSWD}" -keystore "${ROOT}.jks" -gencert -alias "${ROOT}" \ diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 22007352f50af..b09e8d00b70a5 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -49,7 +49,7 @@ ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" # Install binary test dependencies. -RUN for v in "2.7.6" "2.8.0" "2.8.1" "2.9.0" "2.9.1" "2.10.0" "2.11.0" "2.11.1" "2.12.0" "2.13.0" "2.14.0"; \ +RUN for v in "2.7.6" "2.15.0"; \ do cd /opt; \ curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v;\ @@ -67,14 +67,6 @@ RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELE && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT RUN mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME -# Install spark -ARG SPARK_VERSION="2.3.4" -ARG SPARK_NAME="spark-$SPARK_VERSION" -ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7" - -RUN cd /opt && curl -O $APACHE_ARCHIVE/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz -RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME - # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a" @@ -94,7 +86,7 @@ RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMX RUN useradd -ms /bin/bash ducker \ && mkdir -p /home/ducker/ \ && rsync -aiq /root/.ssh/ /home/ducker/.ssh \ - && chown -R ducker /home/ducker/ /mnt/ /var/log/ \ + && chown -R ducker /home/ducker/ /mnt/ /var/log/ /opt \ && echo "LANG=C.UTF-8" >> /home/ducker/.ssh/environment \ && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment \ && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile \ diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py deleted file mode 100644 index bcc200d7b6344..0000000000000 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ /dev/null @@ -1,175 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This module contains spark service class. -""" - -import os.path -from distutils.version import LooseVersion - -from ducktape.cluster.remoteaccount import RemoteCommandError - -from ignitetest.services.utils.ducktests_service import DucktestsService -from ignitetest.services.utils.log_utils import monitor_log -from ignitetest.services.utils.path import PathAware - - -class SparkService(DucktestsService, PathAware): - """ - Start a spark node. - """ - def __init__(self, context, num_nodes=3, version=LooseVersion("2.3.4")): - """ - :param context: test context - :param num_nodes: number of Ignite nodes. - """ - super().__init__(context, num_nodes) - - self.log_level = "DEBUG" - self._version = version - self.init_logs_attribute() - - @property - def product(self): - return "%s-%s" % ("spark", self._version) - - @property - def globals(self): - return self.context.globals - - @property - def config_file(self): - return None - - @property - def log_config_file(self): - return None - - def start(self, **kwargs): - super().start(**kwargs) - - self.logger.info("Waiting for Spark to start...") - - def start_cmd(self, node): - """ - Prepare command to start Spark nodes - """ - if node == self.nodes[0]: - script = "start-master.sh" - else: - script = "start-slave.sh spark://{spark_master}:7077".format(spark_master=self.nodes[0].account.hostname) - - start_script = os.path.join(self.home_dir, "sbin", script) - - cmd = "export SPARK_LOG_DIR={spark_dir}; ".format(spark_dir=self.persistent_root) - cmd += "export SPARK_WORKER_DIR={spark_dir}; ".format(spark_dir=self.persistent_root) - cmd += "{start_script} &".format(start_script=start_script) - - return cmd - - def init_logs_attribute(self): - for node in self.nodes: - self.logs["master_logs" + node.account.hostname] = { - "path": self.master_log_path(node), - "collect_default": True - } - self.logs["worker_logs" + node.account.hostname] = { - "path": self.slave_log_path(node), - "collect_default": True - } - - def start_node(self, node, **kwargs): - self.init_persistent(node) - - cmd = self.start_cmd(node) - self.logger.debug("Attempting to start SparkService on %s with command: %s" % (str(node.account), cmd)) - - if node == self.nodes[0]: - log_file = self.master_log_path(node) - log_msg = "Started REST server for submitting applications" - else: - log_file = self.slave_log_path(node) - log_msg = "Successfully registered with master" - - self.logger.debug("Monitoring - %s" % log_file) - - timeout_sec = 30 - with monitor_log(node, log_file) as monitor: - node.account.ssh(cmd) - monitor.wait_until(log_msg, timeout_sec=timeout_sec, backoff_sec=.1, - err_msg="Spark doesn't start at %d seconds" % timeout_sec) - - if len(self.pids(node)) == 0: - raise Exception("No process ids recorded on node %s" % node.account.hostname) - - def stop_node(self, node, force_stop=False, **kwargs): - if force_stop: - node.account.kill_java_processes(self.java_class_name(node), clean_shutdown=False, allow_fail=True) - else: - if node == self.nodes[0]: - node.account.ssh(os.path.join(self.home_dir, "sbin", "stop-master.sh")) - else: - node.account.ssh(os.path.join(self.home_dir, "sbin", "stop-slave.sh")) - - def clean_node(self, node, **kwargs): - """ - Clean spark persistence files - """ - super().clean_node(node, **kwargs) - - node.account.ssh("rm -rf -- %s" % self.persistent_root, allow_fail=False) - - def pids(self, node): - """ - :return: list of service pids on specific node - """ - try: - cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name(node) - return list(node.account.ssh_capture(cmd, allow_fail=True, callback=int)) - except (RemoteCommandError, ValueError): - return [] - - def java_class_name(self, node): - """ - :param node: Spark node. - :return: Class name depending on node type (master or slave). - """ - if node == self.nodes[0]: - return "org.apache.spark.deploy.master.Master" - - return "org.apache.spark.deploy.worker.Worker" - - def master_log_path(self, node): - """ - :param node: Spark master node. - :return: Path to log file. - """ - return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format( - SPARK_LOG_DIR=self.persistent_root, - userID=node.account.user, - instance=1, - host=node.account.hostname) - - def slave_log_path(self, node): - """ - :param node: Spark slave node. - :return: Path to log file. - """ - return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format( - SPARK_LOG_DIR=self.persistent_root, - userID=node.account.user, - instance=1, - host=node.account.hostname) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index dbbe5787411c8..98b7692d83c1b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -38,13 +38,13 @@ from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.ignite_spec import resolve_spec, SHARED_PREPARED_FILE from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient -from ignitetest.services.utils.jvm_utils import JvmProcessMixin +from ignitetest.services.utils.jvm_utils import JvmProcessMixin, JvmVersionMixin from ignitetest.services.utils.log_utils import monitor_log from ignitetest.services.utils.path import IgnitePathAware from ignitetest.utils.enum import constructible -class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMixin, metaclass=ABCMeta): +class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMixin, JvmVersionMixin, metaclass=ABCMeta): """ The base class to build services aware of Ignite. """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py index 462e513ac0253..e01bb4c7d6efc 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -21,6 +21,7 @@ import re from ignitetest.services.utils.decorators import memoize +from ignitetest.services.utils.jvm_utils import java_version, java_major_version def ignite_jmx_mixin(node, service): @@ -60,13 +61,16 @@ def __init__(self, node): self.node = node self.install_root = node.install_root self.pid = node.pids[0] + self.java_major = java_major_version(java_version(self.node)) @property def jmx_util_cmd(self): """ :return: jmxterm prepared command line invocation. """ - return os.path.join(f"java -jar {self.install_root}/jmxterm.jar -v silent -n") + extra_flag = "--add-exports jdk.jconsole/sun.tools.jconsole=ALL-UNNAMED" if self.java_major >= 15 else "" + + return os.path.join(f"java {extra_flag} -jar {self.install_root}/jmxterm.jar -v silent -n") @memoize def find_mbean(self, pattern, negative_pattern=None, domain='org.apache'): diff --git a/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py index a1175712f4cb0..eaab8566285cb 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py @@ -16,6 +16,9 @@ """ This module contains JVM utilities. """ + +from ignitetest.services.utils.decorators import memoize + DEFAULT_HEAP = "768M" JVM_PARAMS_GC_G1 = "-XX:+UseG1GC -XX:MaxGCPauseMillis=100 " \ @@ -74,6 +77,31 @@ def merge_jvm_settings(src_settings, additionals): return listed +def java_major_version(version): + """ + :param version: Full java version + :return: Java major version + """ + if version: + version = version.split('.') + + return int(version[1]) if version[0] == '1' else int(version[0]) + + return -1 + + +def java_version(node): + """ + :param node: Ducktape cluster node + :return: java version + """ + cmd = r"java -version 2>&1 | awk -F[\"\-] '/version/ {print $2}'" + + raw_version = list(node.account.ssh_capture(cmd, allow_fail=False)) + + return raw_version[0].strip() if raw_version else '' + + def _to_map(params): """""" assert isinstance(params, (str, list)), "JVM params an be string or list only." @@ -107,6 +135,7 @@ class JvmProcessMixin: """ Mixin to work with JVM processes """ + @staticmethod def pids(node, java_class): """ @@ -118,3 +147,15 @@ def pids(node, java_class): cmd = "ps -C java -wwo pid,args | grep '%s' | awk -F' ' '{print $1}'" % java_class return [int(pid) for pid in node.account.ssh_capture(cmd, allow_fail=True)] + + +class JvmVersionMixin: + """ + Mixin to get java version on node. + """ + @memoize + def java_version(self): + """ + :return: Full java version of service. + """ + return java_version(self.nodes[0]) diff --git a/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py b/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py index a6e144db79e08..21ff78bb01c1f 100644 --- a/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py +++ b/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py @@ -22,6 +22,7 @@ from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.jvm_utils import java_major_version from ignitetest.services.utils.ssl.ssl_params import is_ssl_enabled from ignitetest.utils import cluster, ignite_versions, ignore_if from ignitetest.utils.ignite_test import IgniteTest @@ -52,6 +53,10 @@ def upgrade_test(self, versions, ignite_version): "DataLoaderAndCheckerApplication" ) + java_version = service.java_version() + if java_major_version(java_version) > 11: + return f"Skipped on java {java_version}" + for version in versions: service.config = IgniteConfiguration( data_storage=DataStorageConfiguration(default=DataRegionConfiguration(persistence_enabled=True)), diff --git a/modules/ducktests/tests/ignitetest/tests/smoke_test.py b/modules/ducktests/tests/ignitetest/tests/smoke_test.py index bd5d91e15ae04..7ace6ac922821 100644 --- a/modules/ducktests/tests/ignitetest/tests/smoke_test.py +++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py @@ -19,7 +19,6 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService -from ignitetest.services.spark import SparkService from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.services.utils.ignite_configuration import IgniteConfiguration from ignitetest.services.zk.zookeeper import ZookeeperService @@ -67,15 +66,6 @@ def test_ignite_app_start_stop(self, ignite_version): app.stop() ignite.stop() - @cluster(num_nodes=2) - def test_spark_start_stop(self): - """ - Test that SparkService correctly start and stop - """ - spark = SparkService(self.test_context, num_nodes=2) - spark.start() - spark.stop() - @cluster(num_nodes=3) def test_zk_start_stop(self): """ diff --git a/modules/ducktests/tests/ignitetest/utils/version.py b/modules/ducktests/tests/ignitetest/utils/version.py index e886ba938bb3e..df25dbca5fd5f 100644 --- a/modules/ducktests/tests/ignitetest/utils/version.py +++ b/modules/ducktests/tests/ignitetest/utils/version.py @@ -108,7 +108,11 @@ def __repr__(self): V_2_14_0 = IgniteVersion("2.14.0") LATEST_2_14 = V_2_14_0 +# 2.15.x versions +V_2_15_0 = IgniteVersion("2.15.0") +LATEST_2_15 = V_2_15_0 + # if you updated the LATEST version # please check DEV version in 'tests/ignitetest/__init__.py' -LATEST = LATEST_2_14 +LATEST = LATEST_2_15 OLDEST = V_2_7_6