This is an automated email from the ASF dual-hosted git repository.
ivandasch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 04df1ff4a7c IGNITE-20903 [ducktests] Support running on JDK 17 (#11059)
04df1ff4a7c is described below
commit 04df1ff4a7cfe710415818e7f9d3e3af460b1c70
Author: Ivan Daschinskiy <[email protected]>
AuthorDate: Thu Nov 23 12:18:15 2023 +0300
IGNITE-20903 [ducktests] Support running on JDK 17 (#11059)
---
modules/ducktests/tests/certs/functions.sh | 4 +-
modules/ducktests/tests/docker/Dockerfile | 12 +-
.../ducktests/tests/ignitetest/services/spark.py | 175 ---------------------
.../ignitetest/services/utils/ignite_aware.py | 4 +-
.../tests/ignitetest/services/utils/jmx_utils.py | 6 +-
.../tests/ignitetest/services/utils/jvm_utils.py | 41 +++++
.../ignitetest/tests/persistence_upgrade_test.py | 5 +
.../ducktests/tests/ignitetest/tests/smoke_test.py | 10 --
.../ducktests/tests/ignitetest/utils/version.py | 6 +-
9 files changed, 62 insertions(+), 201 deletions(-)
diff --git a/modules/ducktests/tests/certs/functions.sh
b/modules/ducktests/tests/certs/functions.sh
index 9a6235a85d1..0caef590632 100644
--- a/modules/ducktests/tests/certs/functions.sh
+++ b/modules/ducktests/tests/certs/functions.sh
@@ -29,7 +29,7 @@ function makeRoot() {
rm -f "${ALIAS}.pem"
keytool -genkeypair -keystore "${ALIAS}.jks" -alias "${ALIAS}" -dname
"${DNAME}" -ext bc:c -storepass "${PSWD}" \
- -keypass "${PSWD}" -storetype JKS -noprompt -v
+ -keypass "${PSWD}" -storetype JKS -keyalg RSA -noprompt -v
keytool -keystore "${ALIAS}.jks" -storepass "${PSWD}" -keypass "${PSWD}"
-alias "${ALIAS}" -exportcert \
-rfc -file "${ALIAS}.pem" -v
@@ -50,7 +50,7 @@ function makeCA() {
rm -f "${ALIAS}.pem"
keytool -genkeypair -keystore "${ALIAS}.jks" -alias "${ALIAS}" -dname
"${DNAME}" -ext bc:c -storepass "${PSWD}" \
- -keypass "${PSWD}" -storetype JKS -noprompt -v
+ -keypass "${PSWD}" -storetype JKS -keyalg RSA -noprompt -v
keytool -storepass "${PSWD}" -keypass "${PSWD}" -keystore "${ALIAS}.jks"
-certreq -alias "${ALIAS}" \
| keytool -storepass "${PSWD}" -keypass "${PSWD}" -keystore
"${ROOT}.jks" -gencert -alias "${ROOT}" \
diff --git a/modules/ducktests/tests/docker/Dockerfile
b/modules/ducktests/tests/docker/Dockerfile
index 22007352f50..b09e8d00b70 100644
--- a/modules/ducktests/tests/docker/Dockerfile
+++ b/modules/ducktests/tests/docker/Dockerfile
@@ -49,7 +49,7 @@ ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/";
ARG APACHE_ARCHIVE="https://archive.apache.org/dist/";
# Install binary test dependencies.
-RUN for v in "2.7.6" "2.8.0" "2.8.1" "2.9.0" "2.9.1" "2.10.0" "2.11.0"
"2.11.1" "2.12.0" "2.13.0" "2.14.0"; \
+RUN for v in "2.7.6" "2.15.0"; \
do cd /opt; \
curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\
unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin
/opt/ignite-$v;\
@@ -67,14 +67,6 @@ RUN cd /opt && curl -O
$APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELE
&& tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT
RUN mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME
-# Install spark
-ARG SPARK_VERSION="2.3.4"
-ARG SPARK_NAME="spark-$SPARK_VERSION"
-ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7"
-
-RUN cd /opt && curl -O
$APACHE_ARCHIVE/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf
$SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz
-RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME
-
# The version of Kibosh to use for testing.
# If you update this, also update vagrant/base.sh
ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a"
@@ -94,7 +86,7 @@ RUN cd /opt && curl -OL
https://github.com/jiaqi/jmxterm/releases/download/v$JMX
RUN useradd -ms /bin/bash ducker \
&& mkdir -p /home/ducker/ \
&& rsync -aiq /root/.ssh/ /home/ducker/.ssh \
- && chown -R ducker /home/ducker/ /mnt/ /var/log/ \
+ && chown -R ducker /home/ducker/ /mnt/ /var/log/ /opt \
&& echo "LANG=C.UTF-8" >> /home/ducker/.ssh/environment \
&& echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >>
/home/ducker/.ssh/environment \
&& echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile \
diff --git a/modules/ducktests/tests/ignitetest/services/spark.py
b/modules/ducktests/tests/ignitetest/services/spark.py
deleted file mode 100644
index bcc200d7b63..00000000000
--- a/modules/ducktests/tests/ignitetest/services/spark.py
+++ /dev/null
@@ -1,175 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-This module contains spark service class.
-"""
-
-import os.path
-from distutils.version import LooseVersion
-
-from ducktape.cluster.remoteaccount import RemoteCommandError
-
-from ignitetest.services.utils.ducktests_service import DucktestsService
-from ignitetest.services.utils.log_utils import monitor_log
-from ignitetest.services.utils.path import PathAware
-
-
-class SparkService(DucktestsService, PathAware):
- """
- Start a spark node.
- """
- def __init__(self, context, num_nodes=3, version=LooseVersion("2.3.4")):
- """
- :param context: test context
- :param num_nodes: number of Ignite nodes.
- """
- super().__init__(context, num_nodes)
-
- self.log_level = "DEBUG"
- self._version = version
- self.init_logs_attribute()
-
- @property
- def product(self):
- return "%s-%s" % ("spark", self._version)
-
- @property
- def globals(self):
- return self.context.globals
-
- @property
- def config_file(self):
- return None
-
- @property
- def log_config_file(self):
- return None
-
- def start(self, **kwargs):
- super().start(**kwargs)
-
- self.logger.info("Waiting for Spark to start...")
-
- def start_cmd(self, node):
- """
- Prepare command to start Spark nodes
- """
- if node == self.nodes[0]:
- script = "start-master.sh"
- else:
- script = "start-slave.sh
spark://{spark_master}:7077".format(spark_master=self.nodes[0].account.hostname)
-
- start_script = os.path.join(self.home_dir, "sbin", script)
-
- cmd = "export SPARK_LOG_DIR={spark_dir};
".format(spark_dir=self.persistent_root)
- cmd += "export SPARK_WORKER_DIR={spark_dir};
".format(spark_dir=self.persistent_root)
- cmd += "{start_script} &".format(start_script=start_script)
-
- return cmd
-
- def init_logs_attribute(self):
- for node in self.nodes:
- self.logs["master_logs" + node.account.hostname] = {
- "path": self.master_log_path(node),
- "collect_default": True
- }
- self.logs["worker_logs" + node.account.hostname] = {
- "path": self.slave_log_path(node),
- "collect_default": True
- }
-
- def start_node(self, node, **kwargs):
- self.init_persistent(node)
-
- cmd = self.start_cmd(node)
- self.logger.debug("Attempting to start SparkService on %s with
command: %s" % (str(node.account), cmd))
-
- if node == self.nodes[0]:
- log_file = self.master_log_path(node)
- log_msg = "Started REST server for submitting applications"
- else:
- log_file = self.slave_log_path(node)
- log_msg = "Successfully registered with master"
-
- self.logger.debug("Monitoring - %s" % log_file)
-
- timeout_sec = 30
- with monitor_log(node, log_file) as monitor:
- node.account.ssh(cmd)
- monitor.wait_until(log_msg, timeout_sec=timeout_sec,
backoff_sec=.1,
- err_msg="Spark doesn't start at %d seconds" %
timeout_sec)
-
- if len(self.pids(node)) == 0:
- raise Exception("No process ids recorded on node %s" %
node.account.hostname)
-
- def stop_node(self, node, force_stop=False, **kwargs):
- if force_stop:
- node.account.kill_java_processes(self.java_class_name(node),
clean_shutdown=False, allow_fail=True)
- else:
- if node == self.nodes[0]:
- node.account.ssh(os.path.join(self.home_dir, "sbin",
"stop-master.sh"))
- else:
- node.account.ssh(os.path.join(self.home_dir, "sbin",
"stop-slave.sh"))
-
- def clean_node(self, node, **kwargs):
- """
- Clean spark persistence files
- """
- super().clean_node(node, **kwargs)
-
- node.account.ssh("rm -rf -- %s" % self.persistent_root,
allow_fail=False)
-
- def pids(self, node):
- """
- :return: list of service pids on specific node
- """
- try:
- cmd = "jcmd | grep -e %s | awk '{print $1}'" %
self.java_class_name(node)
- return list(node.account.ssh_capture(cmd, allow_fail=True,
callback=int))
- except (RemoteCommandError, ValueError):
- return []
-
- def java_class_name(self, node):
- """
- :param node: Spark node.
- :return: Class name depending on node type (master or slave).
- """
- if node == self.nodes[0]:
- return "org.apache.spark.deploy.master.Master"
-
- return "org.apache.spark.deploy.worker.Worker"
-
- def master_log_path(self, node):
- """
- :param node: Spark master node.
- :return: Path to log file.
- """
- return
"{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format(
- SPARK_LOG_DIR=self.persistent_root,
- userID=node.account.user,
- instance=1,
- host=node.account.hostname)
-
- def slave_log_path(self, node):
- """
- :param node: Spark slave node.
- :return: Path to log file.
- """
- return
"{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format(
- SPARK_LOG_DIR=self.persistent_root,
- userID=node.account.user,
- instance=1,
- host=node.account.hostname)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index dbbe5787411..98b7692d83c 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -38,13 +38,13 @@ from ignitetest.services.utils.background_thread import
BackgroundThreadService
from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue
from ignitetest.services.utils.ignite_spec import resolve_spec,
SHARED_PREPARED_FILE
from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient
-from ignitetest.services.utils.jvm_utils import JvmProcessMixin
+from ignitetest.services.utils.jvm_utils import JvmProcessMixin,
JvmVersionMixin
from ignitetest.services.utils.log_utils import monitor_log
from ignitetest.services.utils.path import IgnitePathAware
from ignitetest.utils.enum import constructible
-class IgniteAwareService(BackgroundThreadService, IgnitePathAware,
JvmProcessMixin, metaclass=ABCMeta):
+class IgniteAwareService(BackgroundThreadService, IgnitePathAware,
JvmProcessMixin, JvmVersionMixin, metaclass=ABCMeta):
"""
The base class to build services aware of Ignite.
"""
diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
index 462e513ac02..e01bb4c7d6e 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
@@ -21,6 +21,7 @@ import os
import re
from ignitetest.services.utils.decorators import memoize
+from ignitetest.services.utils.jvm_utils import java_version,
java_major_version
def ignite_jmx_mixin(node, service):
@@ -60,13 +61,16 @@ class JmxClient:
self.node = node
self.install_root = node.install_root
self.pid = node.pids[0]
+ self.java_major = java_major_version(java_version(self.node))
@property
def jmx_util_cmd(self):
"""
:return: jmxterm prepared command line invocation.
"""
- return os.path.join(f"java -jar {self.install_root}/jmxterm.jar -v
silent -n")
+ extra_flag = "--add-exports
jdk.jconsole/sun.tools.jconsole=ALL-UNNAMED" if self.java_major >= 15 else ""
+
+ return os.path.join(f"java {extra_flag} -jar
{self.install_root}/jmxterm.jar -v silent -n")
@memoize
def find_mbean(self, pattern, negative_pattern=None, domain='org.apache'):
diff --git a/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py
b/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py
index a1175712f4c..eaab8566285 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/jvm_utils.py
@@ -16,6 +16,9 @@
"""
This module contains JVM utilities.
"""
+
+from ignitetest.services.utils.decorators import memoize
+
DEFAULT_HEAP = "768M"
JVM_PARAMS_GC_G1 = "-XX:+UseG1GC -XX:MaxGCPauseMillis=100 " \
@@ -74,6 +77,31 @@ def merge_jvm_settings(src_settings, additionals):
return listed
+def java_major_version(version):
+ """
+ :param version: Full java version
+ :return: Java major version
+ """
+ if version:
+ version = version.split('.')
+
+ return int(version[1]) if version[0] == '1' else int(version[0])
+
+ return -1
+
+
+def java_version(node):
+ """
+ :param node: Ducktape cluster node
+ :return: java version
+ """
+ cmd = r"java -version 2>&1 | awk -F[\"\-] '/version/ {print $2}'"
+
+ raw_version = list(node.account.ssh_capture(cmd, allow_fail=False))
+
+ return raw_version[0].strip() if raw_version else ''
+
+
def _to_map(params):
""""""
assert isinstance(params, (str, list)), "JVM params an be string or list
only."
@@ -107,6 +135,7 @@ class JvmProcessMixin:
"""
Mixin to work with JVM processes
"""
+
@staticmethod
def pids(node, java_class):
"""
@@ -118,3 +147,15 @@ class JvmProcessMixin:
cmd = "ps -C java -wwo pid,args | grep '%s' | awk -F' ' '{print $1}'"
% java_class
return [int(pid) for pid in node.account.ssh_capture(cmd,
allow_fail=True)]
+
+
+class JvmVersionMixin:
+ """
+ Mixin to get java version on node.
+ """
+ @memoize
+ def java_version(self):
+ """
+ :return: Full java version of service.
+ """
+ return java_version(self.nodes[0])
diff --git
a/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py
b/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py
index a6e144db79e..21ff78bb01c 100644
--- a/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/persistence_upgrade_test.py
@@ -22,6 +22,7 @@ from ignitetest.services.ignite_app import
IgniteApplicationService
from ignitetest.services.utils.control_utility import ControlUtility
from ignitetest.services.utils.ignite_configuration import
IgniteConfiguration, DataStorageConfiguration
from ignitetest.services.utils.ignite_configuration.data_storage import
DataRegionConfiguration
+from ignitetest.services.utils.jvm_utils import java_major_version
from ignitetest.services.utils.ssl.ssl_params import is_ssl_enabled
from ignitetest.utils import cluster, ignite_versions, ignore_if
from ignitetest.utils.ignite_test import IgniteTest
@@ -52,6 +53,10 @@ class PersistenceUpgradeTest(IgniteTest):
"DataLoaderAndCheckerApplication"
)
+ java_version = service.java_version()
+ if java_major_version(java_version) > 11:
+ return f"Skipped on java {java_version}"
+
for version in versions:
service.config = IgniteConfiguration(
data_storage=DataStorageConfiguration(default=DataRegionConfiguration(persistence_enabled=True)),
diff --git a/modules/ducktests/tests/ignitetest/tests/smoke_test.py
b/modules/ducktests/tests/ignitetest/tests/smoke_test.py
index bd5d91e15ae..7ace6ac9228 100644
--- a/modules/ducktests/tests/ignitetest/tests/smoke_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py
@@ -19,7 +19,6 @@ This module contains smoke tests that checks that services
work
from ignitetest.services.ignite import IgniteService
from ignitetest.services.ignite_app import IgniteApplicationService
-from ignitetest.services.spark import SparkService
from ignitetest.services.utils.ignite_configuration.discovery import
from_ignite_cluster
from ignitetest.services.utils.ignite_configuration import IgniteConfiguration
from ignitetest.services.zk.zookeeper import ZookeeperService
@@ -67,15 +66,6 @@ class SmokeServicesTest(IgniteTest):
app.stop()
ignite.stop()
- @cluster(num_nodes=2)
- def test_spark_start_stop(self):
- """
- Test that SparkService correctly start and stop
- """
- spark = SparkService(self.test_context, num_nodes=2)
- spark.start()
- spark.stop()
-
@cluster(num_nodes=3)
def test_zk_start_stop(self):
"""
diff --git a/modules/ducktests/tests/ignitetest/utils/version.py
b/modules/ducktests/tests/ignitetest/utils/version.py
index e886ba938bb..df25dbca5fd 100644
--- a/modules/ducktests/tests/ignitetest/utils/version.py
+++ b/modules/ducktests/tests/ignitetest/utils/version.py
@@ -108,7 +108,11 @@ LATEST_2_13 = V_2_13_0
V_2_14_0 = IgniteVersion("2.14.0")
LATEST_2_14 = V_2_14_0
+# 2.15.x versions
+V_2_15_0 = IgniteVersion("2.15.0")
+LATEST_2_15 = V_2_15_0
+
# if you updated the LATEST version
# please check DEV version in 'tests/ignitetest/__init__.py'
-LATEST = LATEST_2_14
+LATEST = LATEST_2_15
OLDEST = V_2_7_6
