IMPALA-8007: Fix test_slow_subscriber Previously, test_slow_subscriber verified a slow subscriber by checking for a precise time window since the last heartbeat. This was non-deterministic since sleep duration in python depends on the machine's workload. This change makes it deterministic by checking for an increase in the time since last heartbeat.
Testing: Executed the test 1000 times in a loop. Change-Id: Ibeed543a145076cd11d5d0e441a257111a66497d Reviewed-on: http://gerrit.cloudera.org:8080/12216 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/85b9c6c4 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/85b9c6c4 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/85b9c6c4 Branch: refs/heads/master Commit: 85b9c6c4257cabb866a3a4580c009f2729d8df8e Parents: a7ea86b Author: poojanilangekar <pooja.nilange...@cloudera.com> Authored: Thu Jan 10 21:39:24 2019 -0800 Committer: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Committed: Sat Jan 12 04:10:15 2019 +0000 ---------------------------------------------------------------------- tests/statestore/test_statestore.py | 38 ++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/85b9c6c4/tests/statestore/test_statestore.py ---------------------------------------------------------------------- diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py index f9e61cf..85f42dc 100644 --- a/tests/statestore/test_statestore.py +++ b/tests/statestore/test_statestore.py @@ -555,23 +555,33 @@ class TestStatestore(): ) def test_slow_subscriber(self): - """Test for IMPALA-6644: This test kills a healthy subscriber and sleeps for a random - interval between 1 and 9 seconds, this lets the heartbeats fail without removing the - subscriber from the set of active subscribers. It then checks the subscribers page - of the statestore to ensure that the 'time_since_heartbeat' field is updated with an - acceptable value. Since the statestore heartbeats at 1 second intervals, an acceptable - value would be between ((sleep_time-1.0), (sleep_time+1.0)).""" + """Test for IMPALA-6644: This test kills a healthy subscriber and sleeps for multiple + intervals of about 1 second each, this lets the heartbeats to the subscriber fail. + It polls the subscribers page of the statestore to ensure that the + 'secs_since_heartbeat' field is updated with an acceptable value. This test only + checks for a strictly increasing value since the actual value of time might depend + on the system load. It stops polling the page once the subscriber is removed from + the set of active subscribers. It also checks that a valid heartbeat record of the + subscriber is found at least once.""" sub = StatestoreSubscriber() sub.start().register().wait_for_heartbeat(1) sub.kill() - sleep_time = randint(1, 9) - time.sleep(sleep_time) - subscribers = get_statestore_subscribers()["subscribers"] - for s in subscribers: - if str(s["id"]) == sub.subscriber_id: - secs_since_heartbeat = float(s["secs_since_heartbeat"]) - assert (secs_since_heartbeat > float(sleep_time - 1.0)) - assert (secs_since_heartbeat < float(sleep_time + 1.0)) + # secs_since_heartbeat is initially unknown. + secs_since_heartbeat = -1 + valid_heartbeat_record = False + while secs_since_heartbeat != 0: + sleep_start_time = time.time() + while time.time() - sleep_start_time < 1: + time.sleep(0.1) + prev_secs_since_heartbeat = secs_since_heartbeat + secs_since_heartbeat = 0 + subscribers = get_statestore_subscribers()["subscribers"] + for s in subscribers: + if str(s["id"]) == sub.subscriber_id: + secs_since_heartbeat = float(s["secs_since_heartbeat"]) + assert (secs_since_heartbeat > prev_secs_since_heartbeat) + valid_heartbeat_record = True + assert valid_heartbeat_record def test_topic_persistence(self): """Test that persistent topic entries survive subscriber failure, but transent topic