This is an automated email from the ASF dual-hosted git repository.
bereng pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-dtest.git
The following commit(s) were added to refs/heads/trunk by this push:
new ec3866d9 Flaky upgrade_through_versions_test
ec3866d9 is described below
commit ec3866d9aec0288b6ac479078e298a05c5df6d23
Author: Bereng <[email protected]>
AuthorDate: Wed Sep 13 08:10:57 2023 +0200
Flaky upgrade_through_versions_test
Patch by Berenguer Blasi; reviewed by Brandon Williams for CASSANDRA-17296
---
upgrade_tests/upgrade_through_versions_test.py | 34 +++++++++++++++++++++++++-
1 file changed, 33 insertions(+), 1 deletion(-)
diff --git a/upgrade_tests/upgrade_through_versions_test.py
b/upgrade_tests/upgrade_through_versions_test.py
index 9fc35fc0..d88249f9 100644
--- a/upgrade_tests/upgrade_through_versions_test.py
+++ b/upgrade_tests/upgrade_through_versions_test.py
@@ -15,7 +15,7 @@ from collections import defaultdict, namedtuple
from multiprocessing import Process, Queue
from queue import Empty, Full
-from cassandra import ConsistencyLevel, WriteTimeout, DriverException
+from cassandra import ConsistencyLevel, WriteTimeout, DriverException,
OperationTimedOut
from cassandra.query import SimpleStatement
from dtest import Tester
@@ -60,6 +60,7 @@ def data_writer(tester, to_verify_queue,
verification_done_queue, rewrite_probab
signal.signal(signal.SIGTERM, handle_sigterm)
+ timeout_retries = 0
while running:
try:
key = None
@@ -86,6 +87,13 @@ def data_writer(tester, to_verify_queue,
verification_done_queue, rewrite_probab
logger.error("Error in data writer process!", dex)
shutdown_gently()
raise
+ except OperationTimedOut:
+ if timeout_retries > 2:
+ shutdown_gently()
+ raise
+ timeout_retries += 1
+ time.sleep(1)
+ continue
except Exception as ex:
logger.error("Error in data writer process!", ex)
shutdown_gently()
@@ -124,6 +132,7 @@ def data_checker(tester, to_verify_queue,
verification_done_queue):
signal.signal(signal.SIGTERM, handle_sigterm)
+ timeout_retries = 0
while running:
try:
# here we could block, but if the writer process terminates early
with an empty queue
@@ -143,6 +152,13 @@ def data_checker(tester, to_verify_queue,
verification_done_queue):
logger.error("Error in data checker process!", dex)
shutdown_gently()
raise
+ except OperationTimedOut:
+ if timeout_retries > 2:
+ shutdown_gently()
+ raise
+ timeout_retries += 1
+ time.sleep(1)
+ continue
except Exception as ex:
logger.error("Error in data checker process!", ex)
shutdown_gently()
@@ -194,6 +210,7 @@ def counter_incrementer(tester, to_verify_queue,
verification_done_queue, rewrit
signal.signal(signal.SIGTERM, handle_sigterm)
+ timeout_retries = 0
while running:
try:
key = None
@@ -219,6 +236,13 @@ def counter_incrementer(tester, to_verify_queue,
verification_done_queue, rewrit
logger.error("Error in counter incrementer process!", dex)
shutdown_gently()
raise
+ except OperationTimedOut:
+ if timeout_retries > 2:
+ shutdown_gently()
+ raise
+ timeout_retries += 1
+ time.sleep(1)
+ continue
except Exception as ex:
logger.error("Error in counter incrementer process!", ex)
shutdown_gently()
@@ -257,6 +281,7 @@ def counter_checker(tester, to_verify_queue,
verification_done_queue):
signal.signal(signal.SIGTERM, handle_sigterm)
+ timeout_retries = 0
while running:
try:
# here we could block, but if the writer process terminates early
with an empty queue
@@ -275,6 +300,13 @@ def counter_checker(tester, to_verify_queue,
verification_done_queue):
logger.error("Error in counter verifier process!", dex)
shutdown_gently()
raise
+ except OperationTimedOut:
+ if timeout_retries > 2:
+ shutdown_gently()
+ raise
+ timeout_retries += 1
+ time.sleep(1)
+ continue
except Exception as ex:
logger.error("Error in counter verifier process!", ex)
shutdown_gently()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
