This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push: new 9125de7 IMPALA-9318: Add admission control setting to cap MT_DOP 9125de7 is described below commit 9125de7ae3d2ba0eca59097fd9732a6fbb609107 Author: Joe McDonnell <joemcdonn...@cloudera.com> AuthorDate: Sat May 16 20:33:49 2020 -0700 IMPALA-9318: Add admission control setting to cap MT_DOP This introduces the max-mt-dop setting for admission control. If a statement runs with an MT_DOP setting that exceeds the max-mt-dop, then the MT_DOP setting is downgraded to the max-mt-dop value. If max-mt-dop is set to a negative value, no limit is applied. max-mt-dop is set via the llama-site.xml and can be set at the daemon level or at the resource pool level. When there is no max-mt-dop setting, it defaults to -1, so no limit is applied. The max-mt-dop is evaluated once prior to query planning. The MT_DOP settings for queries past planning are not reevaluated if the policy changes. If a statement is downgraded, it's runtime profile contains a message explaining the downgrade: MT_DOP limited by admission control: Requested MT_DOP=9 reduced to MT_DOP=4. Testing: - Added custom cluster test with various max-mt-dop settings - Ran core tests Change-Id: I3affb127a5dca517591323f2b1c880aa4b38badd Reviewed-on: http://gerrit.cloudera.org:8080/16020 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- be/src/service/client-request-state.cc | 7 ++++ be/src/service/impala-server.cc | 16 ++++++++ be/src/service/impala-server.h | 5 +++ common/thrift/ImpalaInternalService.thrift | 9 +++++ .../org/apache/impala/util/RequestPoolService.java | 5 +++ fe/src/test/resources/fair-scheduler-maxmtdop.xml | 21 ++++++++++ fe/src/test/resources/llama-site-maxmtdop.xml | 30 ++++++++++++++ .../queries/QueryTest/max-mt-dop.test | 47 ++++++++++++++++++++++ tests/custom_cluster/test_mt_dop.py | 31 +++++++++++++- 9 files changed, 170 insertions(+), 1 deletion(-) diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc index 5a54dbc..c919123 100644 --- a/be/src/service/client-request-state.cc +++ b/be/src/service/client-request-state.cc @@ -196,6 +196,13 @@ Status ClientRequestState::Exec() { DebugQueryOptions(query_ctx_.client_request.query_options)); summary_profile_->AddInfoString("Query Options (set by configuration and planner)", DebugQueryOptions(exec_request_->query_options)); + if (query_ctx_.__isset.overridden_mt_dop_value) { + DCHECK(query_ctx_.client_request.query_options.__isset.mt_dop); + summary_profile_->AddInfoString("MT_DOP limited by admission control", + Substitute("Requested MT_DOP=$0 reduced to MT_DOP=$1", + query_ctx_.overridden_mt_dop_value, + query_ctx_.client_request.query_options.mt_dop)); + } switch (exec_request_->stmt_type) { case TStmtType::QUERY: diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index 3251456..6d0a4dd 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -910,6 +910,9 @@ void ImpalaServer::AddPoolConfiguration(TQueryCtx* ctx, << " overlay_mask=" << overlay_mask.to_string(); OverlayQueryOptions(pool_options, overlay_mask, &ctx->client_request.query_options); + // Enforce the max mt_dop after the defaults and overlays have already been done. + EnforceMaxMtDop(ctx, config.max_mt_dop); + status = ValidateQueryOptions(&pool_options); if (!status.ok()) { VLOG_QUERY << "Ignoring errors while validating default query options for pool=" @@ -917,6 +920,19 @@ void ImpalaServer::AddPoolConfiguration(TQueryCtx* ctx, } } +void ImpalaServer::EnforceMaxMtDop(TQueryCtx* query_ctx, int64_t max_mt_dop) { + TQueryOptions& query_options = query_ctx->client_request.query_options; + // The mt_dop is overridden if all three conditions are met: + // 1. There is a nonnegative max mt_dop setting + // 2. The mt_dop query option is set + // 3. The specified mt_dop is larger than the max mt_dop setting + if (max_mt_dop >= 0 && query_options.__isset.mt_dop && + max_mt_dop < query_options.mt_dop) { + query_ctx->__set_overridden_mt_dop_value(query_options.mt_dop); + query_options.__set_mt_dop(max_mt_dop); + } +} + Status ImpalaServer::Execute(TQueryCtx* query_ctx, shared_ptr<SessionState> session_state, QueryHandle* query_handle) { PrepareQueryContext(query_ctx); diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h index cfe3fc8..91dc3af 100644 --- a/be/src/service/impala-server.h +++ b/be/src/service/impala-server.h @@ -1042,6 +1042,11 @@ class ImpalaServer : public ImpalaServiceIf, void AddPoolConfiguration(TQueryCtx* query_ctx, const QueryOptionsMask& override_options_mask); + /// Helper method to enforce a pool's max mt_dop setting. If the provided maximum is + /// nonnegative and mt_dop is set higher than the maximum, the mt_dop is reduced to the + /// maximum. Otherwise, the mt_dop value is not modified. + void EnforceMaxMtDop(TQueryCtx* query_ctx, int64_t max_mt_dop); + /// Register timeout value upon opening a new session. This will wake up /// session_timeout_thread_. void RegisterSessionTimeout(int32_t timeout); diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift index 097525a..4b055b8 100644 --- a/common/thrift/ImpalaInternalService.thrift +++ b/common/thrift/ImpalaInternalService.thrift @@ -592,6 +592,10 @@ struct TQueryCtx { // Stores the transaction id if the query is transactional. 25: optional i64 transaction_id + + // If mt_dop was overridden by admission control's max mt_dop setting, then this + // is set to the original value. If mt_dop was not overridden, then this is not set. + 26: optional i32 overridden_mt_dop_value } // Descriptor that indicates that a runtime filter is produced by a plan node. @@ -743,6 +747,11 @@ struct TPoolConfig { // runtime to give the maximum memory available across the cluster for the pool. If // this value is zero then it is ignored. 11: required i64 max_memory_multiple = 0; + + // Maximum value for the mt_dop query option. If the mt_dop is set and exceeds this + // maximum, the mt_dop setting is reduced to the maximum. If the max_mt_dop is + // negative, no limit is enforced. + 12: required i64 max_mt_dop = -1; } struct TParseDateStringResult { diff --git a/fe/src/main/java/org/apache/impala/util/RequestPoolService.java b/fe/src/main/java/org/apache/impala/util/RequestPoolService.java index 5f0630d..8850a5c 100644 --- a/fe/src/main/java/org/apache/impala/util/RequestPoolService.java +++ b/fe/src/main/java/org/apache/impala/util/RequestPoolService.java @@ -140,6 +140,9 @@ public class RequestPoolService { private final static String MAX_MEMORY_MULTIPLE = "impala.admission-control.max-memory-multiple"; + // Key for specifying the "Max mt_dop" configuration of the pool + private final static String MAX_MT_DOP = "impala.admission-control.max-mt-dop"; + // String format for a per-pool configuration key. First parameter is the key for the // default, e.g. MAX_PLACED_RESERVATIONS_KEY, and the second parameter is the // pool name. @@ -405,6 +408,8 @@ public class RequestPoolService { getPoolConfigDoubleValue(currentConf, pool, MAX_QUEUED_QUERIES_MULTIPLE, 0.0)); result.setMax_memory_multiple( getPoolConfigValue(currentConf, pool, MAX_MEMORY_MULTIPLE, 0)); + result.setMax_mt_dop( + getPoolConfigValue(currentConf, pool, MAX_MT_DOP, -1)); } if (LOG.isTraceEnabled()) { LOG.debug("getPoolConfig(pool={}): max_mem_resources={}, max_requests={}," diff --git a/fe/src/test/resources/fair-scheduler-maxmtdop.xml b/fe/src/test/resources/fair-scheduler-maxmtdop.xml new file mode 100644 index 0000000..b2d6518 --- /dev/null +++ b/fe/src/test/resources/fair-scheduler-maxmtdop.xml @@ -0,0 +1,21 @@ +<?xml version="1.0"?> +<allocations> + <queue name="root"> + <queue name="negative"> + <aclSubmitApps>*</aclSubmitApps> + </queue> + <queue name="zero"> + <aclSubmitApps>* </aclSubmitApps> + </queue> + <queue name="limited"> + <aclSubmitApps>* </aclSubmitApps> + </queue> + <queue name="largeint"> + <aclSubmitApps>* </aclSubmitApps> + </queue> + <queue name="nosetting"> + <aclSubmitApps>* </aclSubmitApps> + </queue> + <aclSubmitApps> </aclSubmitApps> + </queue> +</allocations> diff --git a/fe/src/test/resources/llama-site-maxmtdop.xml b/fe/src/test/resources/llama-site-maxmtdop.xml new file mode 100644 index 0000000..ab23952 --- /dev/null +++ b/fe/src/test/resources/llama-site-maxmtdop.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<configuration> + <!-- Default values --> + <property> + <name>impala.admission-control.max-mt-dop</name> + <value>8</value> + </property> + + <property> + <name>impala.admission-control.max-mt-dop.root.negative</name> + <value>-1</value> + </property> + + <property> + <name>impala.admission-control.max-mt-dop.root.zero</name> + <value>0</value> + </property> + + <property> + <name>impala.admission-control.max-mt-dop.root.limited</name> + <value>4</value> + </property> + + <property> + <name>impala.admission-control.max-mt-dop.root.largeint</name> + <value>4294967296</value> + </property> + +</configuration> diff --git a/testdata/workloads/functional-query/queries/QueryTest/max-mt-dop.test b/testdata/workloads/functional-query/queries/QueryTest/max-mt-dop.test new file mode 100644 index 0000000..bf082ae --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/max-mt-dop.test @@ -0,0 +1,47 @@ +==== +---- QUERY +# The 'nosetting' resource pool does not have a max-mt-dop setting, so it uses +# the top-level max-mt-dop setting (which is 8). +set request_pool=nosetting; +set mt_dop=9; +select 1; +---- RUNTIME_PROFILE +row_regex: .*Query Options \(set by configuration\): .*MT_DOP=8.* +row_regex: .*MT_DOP limited by admission control: Requested MT_DOP=9 reduced to MT_DOP=8.* +==== +---- QUERY +# The 'limited' resource pool has max-mt-dop set to 4, so the query is downgraded. +set request_pool=limited; +set mt_dop=9; +select 1; +---- RUNTIME_PROFILE +row_regex: .*Query Options \(set by configuration\): .*MT_DOP=4.* +row_regex: .*MT_DOP limited by admission control: Requested MT_DOP=9 reduced to MT_DOP=4.* +==== +---- QUERY +# The 'negative' resource pool has max-mt-dop set to -1, which means the limit is +# disabled. +set request_pool=negative; +set mt_dop=9; +select 1; +---- RUNTIME_PROFILE +row_regex: .*Query Options \(set by configuration\): .*MT_DOP=9.* +==== +---- QUERY +# The 'largeint' resource pool has max-mt-dop set to a value that doesn't fit in 4 bytes. +# The query is not downgraded. +set request_pool=largeint; +set mt_dop=9; +select 1; +---- RUNTIME_PROFILE +row_regex: .*Query Options \(set by configuration\): .*MT_DOP=9.* +==== +---- QUERY +# The 'zero' resource pool has max-mt-dop set to 0, so the query is downgraded to 0. +set request_pool=zero; +set mt_dop=9; +select 1; +---- RUNTIME_PROFILE +row_regex: .*Query Options \(set by configuration\): .*MT_DOP=0.* +row_regex: .*MT_DOP limited by admission control: Requested MT_DOP=9 reduced to MT_DOP=0.* +==== diff --git a/tests/custom_cluster/test_mt_dop.py b/tests/custom_cluster/test_mt_dop.py index cdcce24..fd48e37 100644 --- a/tests/custom_cluster/test_mt_dop.py +++ b/tests/custom_cluster/test_mt_dop.py @@ -15,15 +15,27 @@ # specific language governing permissions and limitations # under the License. +import os import pytest from copy import deepcopy from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.environ import build_flavor_timeout -from tests.common.skip import SkipIfABFS, SkipIfNotHdfsMinicluster +from tests.common.skip import SkipIfNotHdfsMinicluster WAIT_TIME_MS = build_flavor_timeout(60000, slow_build_timeout=100000) +# The path to resources directory which contains the admission control config files +# (used for max mt dop test). +RESOURCES_DIR = os.path.join(os.environ['IMPALA_HOME'], "fe", "src", "test", "resources") + + +def impalad_admission_ctrl_maxmtdop_args(): + fs_allocation_path = os.path.join(RESOURCES_DIR, "fair-scheduler-maxmtdop.xml") + llama_site_path = os.path.join(RESOURCES_DIR, "llama-site-maxmtdop.xml") + return "--llama_site_path={0} --fair_scheduler_allocation_path={1}".format( + llama_site_path, fs_allocation_path) + class TestMtDopFlags(CustomClusterTestSuite): @classmethod @@ -71,3 +83,20 @@ class TestMtDopFlags(CustomClusterTestSuite): vector.get_value('table_format').file_format = 'kudu' self.run_test_case('QueryTest/runtime_filters_mt_dop', vector, test_file_vars={'$RUNTIME_FILTER_WAIT_TIME_MS': str(WAIT_TIME_MS)}) + + +class TestMaxMtDop(CustomClusterTestSuite): + @classmethod + def get_workload(cls): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestMaxMtDop, cls).add_test_dimensions() + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args=impalad_admission_ctrl_maxmtdop_args()) + @SkipIfNotHdfsMinicluster.tuned_for_minicluster + def test_max_mt_dop(self, vector): + self.run_test_case('QueryTest/max-mt-dop', vector)