IMPALA-5104: Admit queries with mem equal to proc mem_limit This allows queries to be admitted with estimated or requested memory equal to the process memory limit.
Added a corresponding test case. Change-Id: I197648f4162f2057141517b4b42ab5196884a65a Reviewed-on: http://gerrit.cloudera.org:8080/7401 Reviewed-by: Matthew Jacobs <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/67bc7a77 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/67bc7a77 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/67bc7a77 Branch: refs/heads/master Commit: 67bc7a774c778c59cdaf7be39ceb5620f75e4f34 Parents: 4655c45 Author: Bikramjeet Vig <[email protected]> Authored: Tue Jul 11 16:39:09 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Jul 12 04:40:51 2017 +0000 ---------------------------------------------------------------------- be/src/scheduling/admission-controller.cc | 2 +- tests/custom_cluster/test_admission_controller.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/67bc7a77/be/src/scheduling/admission-controller.cc ---------------------------------------------------------------------- diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc index 73169f9..543fdf4 100644 --- a/be/src/scheduling/admission-controller.cc +++ b/be/src/scheduling/admission-controller.cc @@ -397,7 +397,7 @@ Status AdmissionController::RejectImmediately(QuerySchedule* schedule, reject_reason = Substitute(REASON_REQ_OVER_POOL_MEM, PrintBytes(cluster_mem_needed), PrintBytes(pool_cfg.max_mem_resources)); } else if (pool_cfg.max_mem_resources > 0 && - schedule->GetPerHostMemoryEstimate() >= GetProcMemLimit()) { + schedule->GetPerHostMemoryEstimate() > GetProcMemLimit()) { reject_reason = Substitute(REASON_REQ_OVER_NODE_MEM, PrintBytes(schedule->GetPerHostMemoryEstimate()), PrintBytes(GetProcMemLimit())); } else if (stats->agg_num_queued() >= pool_cfg.max_queued) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/67bc7a77/tests/custom_cluster/test_admission_controller.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index 5201c75..e84db52 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -315,6 +315,22 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite): assert re.search("Rejected query from pool default-pool : request memory needed " ".* is greater than pool max mem resources 10.00 MB", str(ex)) + # Process mem_limit used in test_mem_limit_upper_bound + PROC_MEM_TEST_LIMIT = 1024 * 1024 * 1024 + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args=impalad_admission_ctrl_flags(1, 1, 10 * PROC_MEM_TEST_LIMIT, + PROC_MEM_TEST_LIMIT)) + def test_mem_limit_upper_bound(self, vector): + """ Test to ensure that a query is admitted if the requested memory is equal to the + process mem limit""" + query = "select * from functional.alltypesagg limit 1" + exec_options = vector.get_value('exec_option') + # Setting requested memory equal to process memory limit + exec_options['mem_limit'] = self.PROC_MEM_TEST_LIMIT + self.execute_query_expect_success(self.client, query, exec_options) + class TestAdmissionControllerStress(TestAdmissionControllerBase): """Submits a number of queries (parameterized) with some delay between submissions (parameterized) and the ability to submit to one impalad or many in a round-robin
