Repository: cassandra-dtest Updated Branches: refs/heads/master e426ce1da -> 6e80b1846
Test corrupting an internode messaging connection, and ensure it reconnects. patch by jasobrown; reviewed by Dinesh Joshi for CASSANDRA-14574 Project: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/commit/6e80b184 Tree: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/tree/6e80b184 Diff: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/diff/6e80b184 Branch: refs/heads/master Commit: 6e80b1846c308bb13d0b700263c89f10caa17d28 Parents: e426ce1 Author: Jason Brown <jasedbr...@gmail.com> Authored: Thu Aug 16 06:27:23 2018 -0700 Committer: Jason Brown <jasedbr...@gmail.com> Committed: Fri Aug 17 05:55:40 2018 -0700 ---------------------------------------------------------------------- byteman/corrupt_internode_messages_gossip.btm | 17 ++++++++ internode_messaging_test.py | 48 ++++++++++++++++++++++ 2 files changed, 65 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/6e80b184/byteman/corrupt_internode_messages_gossip.btm ---------------------------------------------------------------------- diff --git a/byteman/corrupt_internode_messages_gossip.btm b/byteman/corrupt_internode_messages_gossip.btm new file mode 100644 index 0000000..66e4fe2 --- /dev/null +++ b/byteman/corrupt_internode_messages_gossip.btm @@ -0,0 +1,17 @@ +# +# corrupt the first gossip ACK message. we corrupt it on the way out, +# in serialize(), so it fails on deserializing. However, we also need +# to hack the serializedSize(). +# + +RULE corrupt the first gossip ACK message +CLASS org.apache.cassandra.gms.GossipDigestAckSerializer +METHOD serialize(org.apache.cassandra.gms.GossipDigestAck, org.apache.cassandra.io.util.DataOutputPlus, int) +AT ENTRY +# set flag to only run this rule once. +IF NOT flagged("done") +DO + flag("done"); + $2.writeInt(-1); +ENDRULE + http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/6e80b184/internode_messaging_test.py ---------------------------------------------------------------------- diff --git a/internode_messaging_test.py b/internode_messaging_test.py new file mode 100644 index 0000000..d0d4d1f --- /dev/null +++ b/internode_messaging_test.py @@ -0,0 +1,48 @@ +import pytest +import logging +import time + +from dtest import Tester + +since = pytest.mark.since +logger = logging.getLogger(__name__) + +_LOG_ERR_ILLEGAL_CAPACITY = "Caused by: java.lang.IllegalArgumentException: Illegal Capacity: -1" + + +@since('4.0') +class TestInternodeMessaging(Tester): + + @pytest.fixture(autouse=True) + def fixture_add_additional_log_patterns(self, fixture_dtest_setup): + fixture_dtest_setup.ignore_log_patterns = ( + r'Illegal Capacity: -1', + r'reported message size' + ) + + def test_message_corruption(self): + """ + @jira_ticket CASSANDRA-14574 + + Use byteman to corrupt an outgoing gossip ACK message, check that the recipient fails *once* on the message + but does not spin out of control trying to process the rest of the bytes in the buffer. + Then make sure normal messaging can occur after a reconnect (on a different socket, of course). + """ + cluster = self.cluster + cluster.populate(2, install_byteman=True) + cluster.start(wait_other_notice=True) + + node1, node2 = cluster.nodelist() + node1_log_mark = node1.mark_log() + node2_log_mark = node2.mark_log() + + node2.byteman_submit(['./byteman/corrupt_internode_messages_gossip.btm']) + + # wait for the deserialization error to happen on node1 + time.sleep(10) + assert len(node1.grep_log(_LOG_ERR_ILLEGAL_CAPACITY, from_mark=node1_log_mark)) == 1 + + # now, make sure node2 reconnects (and continues gossiping). + # node.watch_log_for() will time out if it cannot find the log entry + assert node2.grep_log('successfully connected to 127.0.0.1:7000 \(GOSSIP\)', + from_mark=node2_log_mark, filename='debug.log') --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
