IMPALA-5838: Improve errors on AC buffer mem rejection The error message returned when a query is rejected due to insufficient buffer memory is misleading. It recommended a mem_limit which would be high enough, but changing the mem_limit may result in changing the plan, which may result in further changes to the buffer memory requirement.
In particular, this can happen when the planner compares the expected hash table size to the mem_limit, and decides to choose a partitioned join over a broadcast join. While we might consider other code changes to improve this, for now lets just be clear in the error message. Testing: * Adds tests that verify the expected behavior with the new error message. Change-Id: I3dc3517195508d86078a8a4b537ae7d2f52fbcb7 Reviewed-on: http://gerrit.cloudera.org:8080/7834 Reviewed-by: Matthew Jacobs <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/77e9e262 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/77e9e262 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/77e9e262 Branch: refs/heads/master Commit: 77e9e262af8a28ecc54aa545ed13f66aa57106a3 Parents: 260a972 Author: Matthew Jacobs <[email protected]> Authored: Fri Aug 25 13:41:05 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Tue Aug 29 02:43:47 2017 +0000 ---------------------------------------------------------------------- be/src/scheduling/admission-controller.cc | 27 ++++++------- .../admission-reject-min-reservation.test | 42 ++++++++++++++++++++ .../custom_cluster/test_admission_controller.py | 23 +---------- 3 files changed, 57 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77e9e262/be/src/scheduling/admission-controller.cc ---------------------------------------------------------------------- diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc index 339935d..1aadf22 100644 --- a/be/src/scheduling/admission-controller.cc +++ b/be/src/scheduling/admission-controller.cc @@ -115,18 +115,19 @@ const string PROFILE_INFO_VAL_QUEUE_DETAIL = "waited $0 ms, reason: $1"; // Error status string details const string REASON_MEM_LIMIT_TOO_LOW_FOR_RESERVATION = "minimum memory reservation is greater than memory available to the query " - "for buffer reservations. Mem available for buffer reservations based on mem_limit: " - "$0, memory reservation needed: $1. Set mem_limit to at least $2. See the query " - "profile for more information."; + "for buffer reservations. Memory reservation needed given the current plan: $0. Set " + "mem_limit to at least $1. Note that changing the mem_limit may also change the " + "plan. See the query profile for more information about the per-node memory " + "requirements."; const string REASON_BUFFER_LIMIT_TOO_LOW_FOR_RESERVATION = "minimum memory reservation is greater than memory available to the query " - "for buffer reservations. Mem available for buffer reservations based on " - "buffer_pool_limit: $0, memory reservation needed: $1. See the query profile for " - "more information."; + "for buffer reservations. Increase the buffer_pool_limit to $0. See the query " + "profile for more information about the per-node memory requirements."; const string REASON_MIN_RESERVATION_OVER_POOL_MEM = - "minimum memory reservation needed is greater than pool max mem resources. pool " - "max mem resources: $0, cluster-wide memory reservation needed: $1. See the query " - "profile for more information."; + "minimum memory reservation needed is greater than pool max mem resources. Pool " + "max mem resources: $0. Cluster-wide memory reservation needed: $1. Increase the " + "pool max mem resources. See the query profile for more information about the " + "per-node memory requirements."; const string REASON_DISABLED_MAX_MEM_RESOURCES = "disabled by pool max mem resources set to 0"; const string REASON_DISABLED_REQUESTS_LIMIT = "disabled by requests limit set to 0"; @@ -415,10 +416,9 @@ bool AdmissionController::RejectImmediately(QuerySchedule* schedule, // Checks related to the min buffer reservation against configured query memory limits: if (schedule->query_options().__isset.buffer_pool_limit && schedule->query_options().buffer_pool_limit > 0) { - const int64_t buffer_pool_limit = schedule->query_options().buffer_pool_limit; - if (max_min_reservation_bytes > buffer_pool_limit) { + if (max_min_reservation_bytes > schedule->query_options().buffer_pool_limit) { *rejection_reason = Substitute(REASON_BUFFER_LIMIT_TOO_LOW_FOR_RESERVATION, - PrintBytes(buffer_pool_limit), PrintBytes(max_min_reservation_bytes)); + PrintBytes(max_min_reservation_bytes)); return true; } } else if (schedule->query_options().__isset.mem_limit && @@ -430,8 +430,7 @@ bool AdmissionController::RejectImmediately(QuerySchedule* schedule, const int64_t required_mem_limit = ReservationUtil::GetMinMemLimitFromReservation(max_min_reservation_bytes); *rejection_reason = Substitute(REASON_MEM_LIMIT_TOO_LOW_FOR_RESERVATION, - PrintBytes(mem_limit), PrintBytes(max_min_reservation_bytes), - PrintBytes(required_mem_limit)); + PrintBytes(max_min_reservation_bytes), PrintBytes(required_mem_limit)); return true; } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77e9e262/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test new file mode 100644 index 0000000..940fd6e --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test @@ -0,0 +1,42 @@ +==== +---- QUERY +set mem_limit=10mb; +select distinct * from functional_parquet.alltypesagg +---- CATCH +minimum memory reservation is greater than memory available to the + query for buffer reservations. Memory reservation needed given the + current plan: 67.00 MB. Set mem_limit to at least 142.00 MB. +==== +---- QUERY +set mem_limit=150mb; +select distinct * from functional_parquet.alltypesagg +---- CATCH +minimum memory reservation needed is greater than pool max mem resources. + Pool max mem resources: 10.00 MB. + Cluster-wide memory reservation needed: 201.00 MB +==== +---- QUERY +set buffer_pool_limit=10mb; +select distinct * from functional_parquet.alltypesagg +---- CATCH +minimum memory reservation is greater than memory available to the query + for buffer reservations. Increase the buffer_pool_limit to 67.00 MB. +==== +---- QUERY +set mem_limit=1024; +select count(*) +from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey +---- CATCH +minimum memory reservation is greater than memory available to the + query for buffer reservations. Memory reservation needed given the + current plan: 4.75 MB. Set mem_limit to at least 79.75 MB. +==== +---- QUERY +set mem_limit=80mb; +select count(*) +from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey +---- CATCH +minimum memory reservation is greater than memory available to the + query for buffer reservations. Memory reservation needed given the + current plan: 17.00 MB. Set mem_limit to at least 92.00 MB. +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77e9e262/tests/custom_cluster/test_admission_controller.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index cf072b9..de97e7c 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -321,7 +321,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite): impalad_args=impalad_admission_ctrl_flags(max_requests=1, max_queued=1, pool_max_mem=10 * 1024 * 1024, proc_mem_limit=1024 * 1024 * 1024), statestored_args=_STATESTORED_ARGS) - def test_reject_min_reservation(self): + def test_reject_min_reservation(self, vector): """Test that the query will be rejected by admission control if: a) the largest per-backend min buffer reservation is larger than the query mem limit @@ -330,26 +330,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite): c) the cluster-wide min-buffer reservation size is larger than the pool memory resources. """ - query = "select distinct * from functional_parquet.alltypesagg" - opts = {'mem_limit': '10MB', 'num_nodes': '1'} - ex = self.execute_query_expect_failure(self.client, query, opts) - assert ("minimum memory reservation is greater than memory available to the query " - "for buffer reservations. Mem available for buffer reservations based on " - "mem_limit: 10.00 MB, memory reservation needed: 34.00 MB. " - "Set mem_limit to at least 109.00 MB.") in str(ex) - - query = "select distinct * from functional_parquet.alltypesagg" - opts = {'buffer_pool_limit': '10MB', 'num_nodes': '1'} - ex = self.execute_query_expect_failure(self.client, query, opts) - assert ("minimum memory reservation is greater than memory available to the query " - "for buffer reservations. Mem available for buffer reservations based on " - "buffer_pool_limit: 10.00 MB, memory reservation needed: 34.00 MB.") in str(ex) - - opts = {'mem_limit': '150MB', 'num_nodes': '1'} - ex = self.execute_query_expect_failure(self.client, query, opts) - assert ("minimum memory reservation needed is greater than pool max mem resources. " - "pool max mem resources: 10.00 MB, cluster-wide memory reservation needed: " - "34.00 MB") in str(ex) + self.run_test_case('QueryTest/admission-reject-min-reservation', vector) # Process mem_limit used in test_mem_limit_upper_bound PROC_MEM_TEST_LIMIT = 1024 * 1024 * 1024
