This is an automated email from the ASF dual-hosted git repository.
brandonwilliams pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-dtest.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8490a74e Complete legacy parsing for upgrade_tests, etc
8490a74e is described below
commit 8490a74ead332761c02a41de7a51d4e989f2db2c
Author: Brandon Williams <[email protected]>
AuthorDate: Wed Sep 21 08:58:13 2022 -0500
Complete legacy parsing for upgrade_tests, etc
patch by brandonwilliams; reviewed by edimitrova for CASSANDRA-17885
---
counter_test.py | 2 +-
dtest.py | 16 ++++++++++++++++
dtest_setup.py | 19 +++++++++----------
legacy_sstables_test.py | 2 +-
offline_tools_test.py | 4 ++--
repair_tests/repair_test.py | 1 +
sstable_generation_loading_test.py | 5 +++--
upgrade_crc_check_chance_test.py | 2 ++
upgrade_internal_auth_test.py | 4 ++--
upgrade_tests/cql_tests.py | 1 +
upgrade_tests/storage_engine_upgrade_test.py | 7 ++++++-
upgrade_tests/thrift_upgrade_test.py | 2 ++
upgrade_tests/upgrade_base.py | 2 ++
upgrade_tests/upgrade_supercolumns_test.py | 4 ++++
upgrade_tests/upgrade_through_versions_test.py | 5 +++++
15 files changed, 57 insertions(+), 19 deletions(-)
diff --git a/counter_test.py b/counter_test.py
index 00fc1df0..aa8215e0 100644
--- a/counter_test.py
+++ b/counter_test.py
@@ -32,9 +32,9 @@ class TestCounters(Tester):
#
cluster.set_install_dir(version='2.0.17')
- self.install_nodetool_legacy_parsing()
cluster.populate(3)
cluster.start()
+ self.install_nodetool_legacy_parsing()
node1, node2, node3 = cluster.nodelist()
diff --git a/dtest.py b/dtest.py
index 01cf4f16..1c295ab4 100644
--- a/dtest.py
+++ b/dtest.py
@@ -402,6 +402,7 @@ def data_size(node, ks, cf):
@param cf: table name
@return: data size in bytes
"""
+ hack_legacy_parsing(node)
cfstats = node.nodetool("cfstats {}.{}".format(ks,cf))[0]
regex = re.compile(r'[\t]')
stats_lines = [regex.sub("", s) for s in cfstats.split('\n')
@@ -517,3 +518,18 @@ def run_scenarios(scenarios, handler,
deferred_exceptions=tuple()):
if errors:
raise MultiError(errors, tracebacks)
+
+def hack_legacy_parsing(node):
+ """ Hack node's shell script for nodetool legacy URL parsing, ala
CASSANDRA-17581 """
+ nodetool = os.path.join(node.get_install_dir(), 'bin', 'nodetool')
+ with open(nodetool, 'r+') as fd:
+ contents = fd.readlines()
+ if "legacy" in contents[len(contents)-6]:
+ logger.debug("nodetool already hacked")
+ elif not contents[len(contents)-5].endswith('\\\n'):
+ logger.debug("version does not appear to need hacking")
+ else:
+ contents.insert(len(contents)-5, "
-Dcom.sun.jndi.rmiURLParsing=legacy \\\n")
+ fd.seek(0)
+ fd.writelines(contents)
+
diff --git a/dtest_setup.py b/dtest_setup.py
index 394dc2f4..1a82555b 100644
--- a/dtest_setup.py
+++ b/dtest_setup.py
@@ -20,7 +20,7 @@ from ccmlib.common import is_win
from ccmlib.cluster import Cluster
from dtest import (get_ip_from_node, make_execution_profile,
get_auth_provider, get_port_from_node,
- get_eager_protocol_version)
+ get_eager_protocol_version, hack_legacy_parsing)
from distutils.version import LooseVersion
from tools.context import log_filter
@@ -84,16 +84,15 @@ class DTestSetup(object):
self.create_cluster_func = None
self.iterations = 0
+ def install_legacy_parsing(self, node):
+ hack_legacy_parsing(node)
+
def install_nodetool_legacy_parsing(self):
- """ Hack nodetool on old versions for legacy URL parsing, ala
CASSANDRA-17581 """
- if self.cluster.version() < LooseVersion('3.0'):
- logger.debug("hacking nodetool for legacy parsing")
- nodetool = os.path.join(self.cluster.get_install_dir(), 'bin',
'nodetool')
- with open(nodetool, 'r+') as fd:
- contents = fd.readlines()
- contents.insert(len(contents)-5, "
-Dcom.sun.jndi.rmiURLParsing=legacy \\\n")
- fd.seek(0)
- fd.writelines(contents)
+ """ Install nodetool legacy parsing on the cluster """
+ if self.cluster.version() < LooseVersion('3.11.13'):
+ logger.debug("hacking nodetool for legacy parsing on
{}".format(self.cluster.version()))
+ for node in self.cluster.nodelist():
+ self.install_legacy_parsing(node)
else:
logger.debug("not modifying nodetool on version
{}".format(self.cluster.version()))
diff --git a/legacy_sstables_test.py b/legacy_sstables_test.py
index c09e4dbf..477e2e37 100644
--- a/legacy_sstables_test.py
+++ b/legacy_sstables_test.py
@@ -27,9 +27,9 @@ class TestLegacySSTables(Tester):
# start with 2.1.20 to generate a legacy sstable
cluster.set_install_dir(version='2.1.20')
- self.install_nodetool_legacy_parsing()
cluster.populate(1).start()
+ self.install_nodetool_legacy_parsing()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
diff --git a/offline_tools_test.py b/offline_tools_test.py
index 1c3d81bc..d8a0e73e 100644
--- a/offline_tools_test.py
+++ b/offline_tools_test.py
@@ -362,12 +362,10 @@ class TestOfflineTools(Tester):
elif testversion < '3.0':
logger.debug('Test version: {} - installing
github:apache/cassandra-2.1'.format(testversion))
cluster.set_install_dir(version='github:apache/cassandra-2.1')
- self.install_nodetool_legacy_parsing()
# As of 3.5, sstable format 'ma' from 3.0 is still the latest -
install 2.2 to upgrade from
elif testversion < '4.0':
logger.debug('Test version: {} - installing
github:apache/cassandra-2.2'.format(testversion))
cluster.set_install_dir(version='github:apache/cassandra-2.2')
- self.install_nodetool_legacy_parsing()
# From 4.0, one can only upgrade from 3.0
else:
logger.debug('Test version: {} - installing
github:apache/cassandra-3.0'.format(testversion))
@@ -375,6 +373,7 @@ class TestOfflineTools(Tester):
# Start up last major version, write out an sstable to upgrade, and
stop node
cluster.populate(1).start()
+ self.install_nodetool_legacy_parsing()
[node1] = cluster.nodelist()
# Check that node1 is actually what we expect
logger.debug('Downgraded install dir:
{}'.format(node1.get_install_dir()))
@@ -392,6 +391,7 @@ class TestOfflineTools(Tester):
logger.debug('Upgraded to original install dir:
{}'.format(node1.get_install_dir()))
# Perform a node start/stop so system tables get internally updated,
otherwise we may get "Unknown keyspace/table ks.cf"
cluster.start()
+ self.install_nodetool_legacy_parsing()
node1.flush()
cluster.stop()
diff --git a/repair_tests/repair_test.py b/repair_tests/repair_test.py
index 29f46e90..6c31f6b1 100644
--- a/repair_tests/repair_test.py
+++ b/repair_tests/repair_test.py
@@ -110,6 +110,7 @@ class BaseRepairTest(Tester):
# Insert 1000 keys, kill node 3, insert 1 key, restart node 3, insert
1000 more keys
logger.debug("Inserting data...")
insert_c1c2(session, n=1000, consistency=ConsistencyLevel.ALL, ks='ks')
+ self.install_legacy_parsing(node3)
node3.flush()
node3.stop(wait_other_notice=True)
insert_c1c2(session, keys=(1000, ), consistency=ConsistencyLevel.TWO,
ks='ks')
diff --git a/sstable_generation_loading_test.py
b/sstable_generation_loading_test.py
index 5c309b2c..1970821a 100644
--- a/sstable_generation_loading_test.py
+++ b/sstable_generation_loading_test.py
@@ -174,10 +174,11 @@ class BaseSStableLoaderTester(Tester):
default_install_dir = self.cluster.get_install_dir()
# Forcing cluster version on purpose
cluster.set_install_dir(version=self.upgrade_from)
- self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
+ self.install_nodetool_legacy_parsing()
logger.debug("Using jvm_args={}".format(self.jvm_args))
cluster.populate(2).start(jvm_args=list(self.jvm_args))
+ self.install_nodetool_legacy_parsing()
node1, node2 = cluster.nodelist()
time.sleep(.5)
@@ -216,9 +217,9 @@ class BaseSStableLoaderTester(Tester):
logger.debug("Running sstableloader with version from %s" %
(default_install_dir))
# Return to previous version
cluster.set_install_dir(install_dir=default_install_dir)
- self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
+ self.install_nodetool_legacy_parsing()
cluster.start(jvm_args=list(self.jvm_args))
time.sleep(5) # let gossip figure out what is going on
diff --git a/upgrade_crc_check_chance_test.py b/upgrade_crc_check_chance_test.py
index 88f590a7..3c65f395 100644
--- a/upgrade_crc_check_chance_test.py
+++ b/upgrade_crc_check_chance_test.py
@@ -123,6 +123,7 @@ class TestCrcCheckChanceUpgrade(Tester):
def upgrade_to_version(self, tag, node):
format_args = {'node': node.name, 'tag': tag}
logger.debug('Upgrading node {node} to {tag}'.format(**format_args))
+ self.install_legacy_parsing(node)
# drain and shutdown
node.drain()
node.watch_log_for("DRAINED")
@@ -134,6 +135,7 @@ class TestCrcCheckChanceUpgrade(Tester):
logger.debug('Set new cassandra dir for {node}:
{tag}'.format(**format_args))
node.set_install_dir(version='git:' + tag, verbose=True)
+ self.install_legacy_parsing(node)
# Restart node on new version
logger.debug('Starting {node} on new version
({tag})'.format(**format_args))
# Setup log4j / logback again (necessary moving from 2.0 -> 2.1):
diff --git a/upgrade_internal_auth_test.py b/upgrade_internal_auth_test.py
index 64b580f1..395f0ace 100644
--- a/upgrade_internal_auth_test.py
+++ b/upgrade_internal_auth_test.py
@@ -58,9 +58,9 @@ class TestAuthUpgrade(Tester):
# Forcing cluster version on purpose
cluster.set_install_dir(version="2.1.16")
- self.install_nodetool_legacy_parsing()
cluster.populate(3).start()
+ self.install_nodetool_legacy_parsing()
node1, node2, node3 = cluster.nodelist()
@@ -121,8 +121,8 @@ class TestAuthUpgrade(Tester):
# Forcing cluster version on purpose
cluster.set_install_dir(version="github:apache/cassandra-2.1")
- self.install_nodetool_legacy_parsing()
cluster.populate(3).start()
+ self.install_nodetool_legacy_parsing()
node1, node2, node3 = cluster.nodelist()
diff --git a/upgrade_tests/cql_tests.py b/upgrade_tests/cql_tests.py
index 8c3cd975..e29042f8 100644
--- a/upgrade_tests/cql_tests.py
+++ b/upgrade_tests/cql_tests.py
@@ -2722,6 +2722,7 @@ class TestCQL(UpgradeTester):
execute_concurrent_with_args(cursor,
cursor.prepare("INSERT INTO t1 (a, b)
VALUES (?, ?)"),
[(i, i) for i in range(100)])
+ self.install_nodetool_legacy_parsing()
self.cluster.flush()
def check_read_all(cursor):
diff --git a/upgrade_tests/storage_engine_upgrade_test.py
b/upgrade_tests/storage_engine_upgrade_test.py
index ebedcb9a..93805c35 100644
--- a/upgrade_tests/storage_engine_upgrade_test.py
+++ b/upgrade_tests/storage_engine_upgrade_test.py
@@ -43,7 +43,7 @@ class TestStorageEngineUpgrade(Tester):
cluster.set_install_dir(version=indev_3_0_x.version)
else:
cluster.set_install_dir(version=indev_2_1_x.version)
- self.install_nodetool_legacy_parsing()
+ self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
cluster.populate(1).start()
@@ -57,6 +57,7 @@ class TestStorageEngineUpgrade(Tester):
def _do_upgrade(self, login_keyspace=True):
cluster = self.cluster
+ self.install_nodetool_legacy_parsing()
node1 = cluster.nodelist()[0]
node1.flush()
@@ -64,10 +65,12 @@ class TestStorageEngineUpgrade(Tester):
node1.stop(wait_other_notice=True)
node1.set_install_dir(install_dir=self.fixture_dtest_setup.default_install_dir)
+ self.install_legacy_parsing(node1)
node1.start(wait_for_binary_proto=True)
if self.fixture_dtest_setup.bootstrap:
cluster.set_install_dir(install_dir=self.fixture_dtest_setup.default_install_dir)
+ self.install_nodetool_legacy_parsing()
# Add a new node, bootstrap=True ensures that it is not a seed
node2 = new_node(cluster, bootstrap=True)
node2.start(wait_for_binary_proto=True,
jvm_args=self.fixture_dtest_setup.jvm_args)
@@ -278,6 +281,7 @@ class TestStorageEngineUpgrade(Tester):
ROWS = 100
session = self._setup_cluster()
+ self.install_nodetool_legacy_parsing()
session.execute('CREATE TABLE t (k int, t int, v1 int, v2 blob, v3
set<int>, PRIMARY KEY (k, t))')
@@ -338,6 +342,7 @@ class TestStorageEngineUpgrade(Tester):
for r in range(ROWS):
session.execute("INSERT INTO t(k, t, v1, v2) VALUES ({}, {},
{}, {})".format(p, r, r % 2, r * 2))
+ self.install_nodetool_legacy_parsing()
self.cluster.flush()
assert_all(session,
diff --git a/upgrade_tests/thrift_upgrade_test.py
b/upgrade_tests/thrift_upgrade_test.py
index 1b29862c..f0f0b015 100644
--- a/upgrade_tests/thrift_upgrade_test.py
+++ b/upgrade_tests/thrift_upgrade_test.py
@@ -278,6 +278,7 @@ class TestUpgradeSuperColumnsThrough(Tester):
def test_dense_supercolumn(self):
cluster = self.prepare()
+ self.install_nodetool_legacy_parsing()
node = self.cluster.nodelist()[0]
node.nodetool("enablethrift")
cursor = self.patient_cql_connection(node, row_factory=dict_factory)
@@ -325,6 +326,7 @@ class TestUpgradeSuperColumnsThrough(Tester):
def test_sparse_supercolumn(self):
cluster = self.prepare()
+ self.install_nodetool_legacy_parsing()
node = self.cluster.nodelist()[0]
node.nodetool("enablethrift")
cursor = self.patient_cql_connection(node, row_factory=dict_factory)
diff --git a/upgrade_tests/upgrade_base.py b/upgrade_tests/upgrade_base.py
index f23fe982..dbd2fc57 100644
--- a/upgrade_tests/upgrade_base.py
+++ b/upgrade_tests/upgrade_base.py
@@ -151,6 +151,7 @@ class UpgradeTester(Tester, metaclass=ABCMeta):
returned instead.
"""
session.cluster.shutdown()
+ self.install_nodetool_legacy_parsing()
node1 = self.cluster.nodelist()[0]
node2 = self.cluster.nodelist()[1]
@@ -172,6 +173,7 @@ class UpgradeTester(Tester, metaclass=ABCMeta):
switch_jdks(self.UPGRADE_PATH.upgrade_meta.java_version)
node1.set_install_dir(version=self.UPGRADE_PATH.upgrade_version)
+ self.install_legacy_parsing(node1)
# this is a bandaid; after refactoring, upgrades should account for
protocol version
new_version_from_build =
get_version_from_build(node1.get_install_dir())
diff --git a/upgrade_tests/upgrade_supercolumns_test.py
b/upgrade_tests/upgrade_supercolumns_test.py
index af22e803..bca02305 100644
--- a/upgrade_tests/upgrade_supercolumns_test.py
+++ b/upgrade_tests/upgrade_supercolumns_test.py
@@ -59,6 +59,7 @@ class TestSCUpgrade(Tester):
# Forcing cluster version on purpose
cluster.set_install_dir(version=cassandra_version)
+ self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
if "memtable_allocation_type" in cluster._config_options:
del cluster._config_options['memtable_allocation_type']
@@ -89,6 +90,7 @@ class TestSCUpgrade(Tester):
def _upgrade_super_columns_through_versions_test(self, upgrade_path):
cluster = self.prepare()
+ self.install_nodetool_legacy_parsing()
node1 = cluster.nodelist()[0]
node1.run_cqlsh(cmds="""CREATE KEYSPACE supcols WITH replication = {
'class': 'SimpleStrategy',
@@ -161,8 +163,10 @@ class TestSCUpgrade(Tester):
# Update Cassandra Directory
for node in nodes:
node.set_install_dir(version=tag)
+ self.install_nodetool_legacy_parsing()
logger.debug("Set new cassandra dir for %s: %s" % (node.name,
node.get_install_dir()))
self.cluster.set_install_dir(version=tag)
+ self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
for node in nodes:
if tag < "2.1":
diff --git a/upgrade_tests/upgrade_through_versions_test.py
b/upgrade_tests/upgrade_through_versions_test.py
index 47f54281..e4f8a17c 100644
--- a/upgrade_tests/upgrade_through_versions_test.py
+++ b/upgrade_tests/upgrade_through_versions_test.py
@@ -327,6 +327,7 @@ class TestUpgrade(Tester):
.format(self.test_version_metas[0].version,
self.test_version_metas[0].java_version))
cluster = self.cluster
cluster.set_install_dir(version=self.test_version_metas[0].version)
+ self.install_nodetool_legacy_parsing()
switch_jdks(self.test_version_metas[0].java_version)
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
logger.debug("Versions to test (%s): %s" % (type(self), str([v.version
for v in self.test_version_metas])))
@@ -435,6 +436,7 @@ class TestUpgrade(Tester):
(num + 1, len(self.cluster.nodelist()),
version_meta.version))
self.cluster.set_install_dir(version=version_meta.version)
+ self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
# Stop write processes
@@ -454,6 +456,7 @@ class TestUpgrade(Tester):
self.upgrade_to_version(version_meta,
internode_ssl=internode_ssl)
self.cluster.set_install_dir(version=version_meta.version)
+ self.install_nodetool_legacy_parsing()
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
self._check_values()
@@ -511,6 +514,7 @@ class TestUpgrade(Tester):
if not partial:
nodes = self.cluster.nodelist()
+ self.install_nodetool_legacy_parsing()
for node in nodes:
logger.debug('Shutting down node: ' + node.name)
node.drain()
@@ -519,6 +523,7 @@ class TestUpgrade(Tester):
for node in nodes:
node.set_install_dir(version=version_meta.version)
+ self.install_legacy_parsing(node)
logger.debug("Set new cassandra dir for %s: %s" % (node.name,
node.get_install_dir()))
if internode_ssl and (LooseVersion(version_meta.family) >=
CASSANDRA_4_0):
node.set_configuration_options({'server_encryption_options':
{'enabled': True, 'enable_legacy_ssl_storage_port': True}})
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
