IMPALA-3200: More end-to-end tests * Test that query startup fails if there is insufficient reservation on one backend. * Verify that free buffers and clean pages shrink to zero when idle.
Change-Id: I3a3b6adf5b97b16594c6c4855d5fa831c5a90e10 Reviewed-on: http://gerrit.cloudera.org:8080/7668 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f74f665b Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f74f665b Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f74f665b Branch: refs/heads/master Commit: f74f665b6cf4425ad890c9e1e79aced883ccbf7a Parents: e4a0e2f Author: Tim Armstrong <[email protected]> Authored: Mon Aug 14 13:57:22 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Aug 16 08:15:30 2017 +0000 ---------------------------------------------------------------------- tests/custom_cluster/test_mem_reservations.py | 55 ++++++++++++++++++---- tests/verifiers/metric_verifier.py | 6 ++- 2 files changed, 52 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f74f665b/tests/custom_cluster/test_mem_reservations.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_mem_reservations.py b/tests/custom_cluster/test_mem_reservations.py index d8a19e8..bf1641b 100644 --- a/tests/custom_cluster/test_mem_reservations.py +++ b/tests/custom_cluster/test_mem_reservations.py @@ -19,6 +19,8 @@ import pytest import threading from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.impala_test_suite import LOG +from tests.verifiers.metric_verifier import MetricVerifier class TestMemReservations(CustomClusterTestSuite): """Tests for memory reservations that require custom cluster arguments.""" @@ -28,7 +30,8 @@ class TestMemReservations(CustomClusterTestSuite): return 'functional-query' @pytest.mark.execute_serially - @CustomClusterTestSuite.with_args(impalad_args="--buffer_pool_limit=2g") + @CustomClusterTestSuite.with_args( + impalad_args="--buffer_pool_limit=2g --memory_maintenance_sleep_time_ms=100") def test_per_backend_min_reservation(self, vector): """Tests that the per-backend minimum reservations are used (IMPALA-4833). The test sets the buffer_pool_limit very low (2gb), and then runs a query against @@ -37,12 +40,15 @@ class TestMemReservations(CustomClusterTestSuite): per-backend minimum reservations are not used, then one of the queries fails to acquire its minimum reservation. This was verified to fail before IMPALA-4833, and succeeds after. + + Memory maintenance sleep time is set low so we can verify that buffers are + released. """ assert len(self.cluster.impalads) == 3 # This query will have scan fragments on all nodes, but the coordinator fragment # has 6 analytic nodes, 5 sort nodes, and an aggregation. - QUERY = """ + COORDINATOR_QUERY = """ select max(t.c1), avg(t.c2), min(t.c3), avg(c4), avg(c5), avg(c6) from (select max(tinyint_col) over (order by int_col) c1, @@ -54,15 +60,22 @@ class TestMemReservations(CustomClusterTestSuite): from functional.alltypes) t; """ - # Set the DEFAULT_SPILLABLE_BUFFER_SIZE and MIN_SPILLABLE_BUFFER_SIZE to 64MiB - # so that the coordinator node requires ~1.2gb and the other backends require ~200mb. + # This query has two grouping aggregations on each node. + SYMMETRIC_QUERY = """ + select count(*) + from (select distinct * from functional.alltypes) v""" + + # Set the DEFAULT_SPILLABLE_BUFFER_SIZE and MIN_SPILLABLE_BUFFER_SIZE to 64MiB. + # so that for COORDINATOR_QUERY, the coordinator node requires ~1.2gb and the + # other backends require ~200mb and for SYMMETRIC_QUERY all backends require + # ~1.05gb. CONFIG_MAP = {'DEFAULT_SPILLABLE_BUFFER_SIZE': '67108864', 'MIN_SPILLABLE_BUFFER_SIZE': '67108864'} - # Create two threads to submit QUERY to two different coordinators concurrently. class QuerySubmitThread(threading.Thread): - def __init__(self, coordinator): + def __init__(self, query, coordinator): super(QuerySubmitThread, self).__init__() + self.query = query self.coordinator = coordinator self.error = None @@ -71,7 +84,7 @@ class TestMemReservations(CustomClusterTestSuite): try: client.set_configuration(CONFIG_MAP) for i in xrange(20): - result = client.execute(QUERY) + result = client.execute(self.query) assert result.success assert len(result.data) == 1 except Exception, e: @@ -79,8 +92,34 @@ class TestMemReservations(CustomClusterTestSuite): finally: client.close() - threads = [QuerySubmitThread(self.cluster.impalads[i]) for i in xrange(2)] + # Create two threads to submit COORDINATOR_QUERY to two different coordinators concurrently. + # They should both succeed. + threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[i]) + for i in xrange(2)] for t in threads: t.start() for t in threads: t.join() assert t.error is None + + # Create two threads to submit COORDINATOR_QUERY to one coordinator and + # SYMMETRIC_QUERY to another coordinator. One of the queries should fail because + # memory would be overcommitted on daemon 0. + threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[0]), + QuerySubmitThread(SYMMETRIC_QUERY, self.cluster.impalads[1])] + for t in threads: t.start() + num_errors = 0 + for t in threads: + t.join() + if t.error is not None: + assert "Failed to get minimum memory reservation" in t.error + LOG.info("Query failed with error: %s", t.error) + LOG.info(t.query) + num_errors += 1 + assert num_errors == 1 + + # Check that free buffers are released over time. We set the memory maintenance sleep + # time very low above so this should happen quickly. + verifiers = [MetricVerifier(i.service) for i in self.cluster.impalads] + for v in verifiers: + v.wait_for_metric("buffer-pool.free-buffers", 0, timeout=60) + v.wait_for_metric("buffer-pool.free-buffer-bytes", 0, timeout=60) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f74f665b/tests/verifiers/metric_verifier.py ---------------------------------------------------------------------- diff --git a/tests/verifiers/metric_verifier.py b/tests/verifiers/metric_verifier.py index d5753fd..3331135 100644 --- a/tests/verifiers/metric_verifier.py +++ b/tests/verifiers/metric_verifier.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. # -# Verfier for common impalad metrics +# Verifier for common impalad metrics # List of metrics that should be equal to zero when there are no outstanding queries. METRIC_LIST = [ @@ -30,6 +30,10 @@ METRIC_LIST = [ "impala-server.num-files-open-for-insert", # Disable checking of num-missing-volume-id due to IMPALA-467 # "impala-server.scan-ranges.num-missing-volume-id", + # Buffer pool pages belong to specific queries. Therefore there should be + # no clean pages if there are no queries running. + "buffer-pool.clean-pages", + "buffer-pool.clean-page-bytes" ] class MetricVerifier(object):
