Repository: incubator-impala Updated Branches: refs/heads/master 0ad935b63 -> f60b2beb8
IMPALA-3790: AC tests timeout in codecoverage builds The codecoverage builds are often timing out in the admission control stress tests. I don't believe there to be any admission control issue, just the codecoverage overhead seems to be very, very high, especially under the concurrent load generated by this tool. This increases the timeout significantly. Change-Id: I9e844cd9fe31464eb410707ae7ef7c71f492f129 Reviewed-on: http://gerrit.cloudera.org:8080/3794 Reviewed-by: Michael Brown <[email protected]> Reviewed-by: Matthew Jacobs <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f60b2beb Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f60b2beb Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f60b2beb Branch: refs/heads/master Commit: f60b2beb8d2a7383b41ea2a34f10f1732362c438 Parents: 0ad935b Author: Matthew Jacobs <[email protected]> Authored: Thu Jul 14 15:26:18 2016 -0700 Committer: Internal Jenkins <[email protected]> Committed: Wed Jul 27 22:43:45 2016 +0000 ---------------------------------------------------------------------- .../custom_cluster/test_admission_controller.py | 24 ++++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f60b2beb/tests/custom_cluster/test_admission_controller.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index b4b6015..0983b20 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -11,6 +11,7 @@ from time import sleep, time from tests.beeswax.impala_beeswax import ImpalaBeeswaxException from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import specific_build_type_timeout from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.test_dimensions import ( create_single_exec_option_dimension, @@ -20,7 +21,6 @@ from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session from ImpalaService import ImpalaHiveServer2Service from TCLIService import TCLIService - LOG = logging.getLogger('admission_test') # We set a WAIT debug action so it doesn't complete the execution of this query. The @@ -49,6 +49,10 @@ POOL_NAME = "default-pool" # The statestore heartbeat and topic update frequency (ms). Set low for testing. STATESTORE_RPC_FREQUENCY_MS = 500 +# Stress test timeout (seconds). The timeout needs to be significantly higher in code +# coverage builds (IMPALA-3790). +STRESS_TIMEOUT = specific_build_type_timeout(30, code_coverage_build_timeout=600) + # The number of queries that can execute concurrently in the pool POOL_NAME. MAX_NUM_CONCURRENT_QUERIES = 5 @@ -379,7 +383,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase): metric_key(self.pool_name, 'total-%s' % short_name), 0) return metrics - def wait_for_metric_changes(self, metric_names, initial, expected_delta, timeout=30): + def wait_for_metric_changes(self, metric_names, initial, expected_delta): """ Waits for the sum of metrics in metric_names to change by at least expected_delta. @@ -408,11 +412,11 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase): LOG.debug("Found all %s metrics after %s seconds", delta_sum, round(time() - start_time, 1)) return (deltas, current) - assert (time() - start_time < timeout),\ - "Timed out waiting %s seconds for metrics" % (timeout,) + assert (time() - start_time < STRESS_TIMEOUT),\ + "Timed out waiting %s seconds for metrics" % (STRESS_TIMEOUT,) sleep(1) - def wait_for_statestore_updates(self, heartbeats, timeout=30): + def wait_for_statestore_updates(self, heartbeats): """Waits for a number of statestore heartbeats from all impalads.""" start_time = time() num_impalads = len(self.impalads) @@ -430,12 +434,12 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase): for impalad in self.impalads: curr[impalad] = impalad.service.get_metric_value(\ 'statestore-subscriber.topic-update-interval-time')['count'] - assert (time() - start_time < timeout),\ - "Timed out waiting %s seconds for heartbeats" % (timeout,) + assert (time() - start_time < STRESS_TIMEOUT),\ + "Timed out waiting %s seconds for heartbeats" % (STRESS_TIMEOUT,) sleep(STATESTORE_RPC_FREQUENCY_MS / float(1000)) LOG.debug("Waited %s for %s heartbeats", round(time() - start_time, 1), heartbeats) - def wait_for_admitted_threads(self, num_threads, timeout=30): + def wait_for_admitted_threads(self, num_threads): """ Wait for query submission threads to update after being admitted, as determined by observing metric changes. This is necessary because the metrics may change @@ -448,9 +452,9 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase): # lock to synchronize before checking the list length (on which another thread # may call append() concurrently). while len(self.executing_threads) < num_threads: - assert (time() - start_time < timeout),\ + assert (time() - start_time < STRESS_TIMEOUT),\ "Timed out waiting %s seconds for %s admitted client rpcs to return" %\ - (timeout, num_threads) + (STRESS_TIMEOUT, num_threads) sleep(0.1) LOG.debug("Found all %s admitted threads after %s seconds", num_threads, round(time() - start_time, 1))
