Repository: impala Updated Branches: refs/heads/master 315142190 -> 2e6034786
IMPALA-3271: organise and warn on removed startup flags Gather all the removed flags in a single place and establish consistent behaviour. If the flag is set, a warning is logged. GFlags are initialised before logging, so the warning goes to stderr instead of the WARNING log. The affected flags are: be_service_threads, cgroup_hierarchy_path, enable_accept_queue_server, enable_partitioned_aggregation, enable_partitioned_hash_join, enable_phj_probe_side_filtering, enable_rm, llama_addresses, llama_callback_port, llama_host, llama_max_request_attempts, llama_port, llama_registration_timeout_secs, llama_registration_wait_secs, local_nodemanager_url, resource_broker_cnxn_attempts, resource_broker_cnxn_retry_interval_ms, resource_broker_recv_timeout, resource_broker_send_timeout, rm_always_use_defaults, rm_default_cpu_vcores, rm_default_memory, rpc_cnxn_attempts, rpc_cnxn_retry_interval_ms, staging_cgroup, suppress_unknown_disk_id_warnings, use_statestore, Testing: Ran "start-impala-cluster.py --impalad_args=--enable_rm=true" and verified that the warning appeared in logs/cluster/impalad-error.log Cherry-picks: not for 2.x Change-Id: Ic9bb4792e1b18840aa85aa5d755a09f2b5461f34 Reviewed-on: http://gerrit.cloudera.org:8080/9173 Reviewed-by: Tim Armstrong <tarmstr...@cloudera.com> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0047f813 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0047f813 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0047f813 Branch: refs/heads/master Commit: 0047f813abc4dc8fb08a15ac8b9d0f582ab8a5f5 Parents: 3151421 Author: Tim Armstrong <tarmstr...@cloudera.com> Authored: Wed Jan 31 15:13:04 2018 -0800 Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org> Committed: Fri Feb 9 20:01:33 2018 +0000 ---------------------------------------------------------------------- be/src/common/global-flags.cc | 56 +++++++++++++++++++++++++- be/src/exec/exec-node.cc | 2 - be/src/exec/hdfs-scan-node-base.cc | 3 -- be/src/exec/partitioned-hash-join-node.cc | 2 - be/src/rpc/thrift-server.cc | 3 -- be/src/runtime/exec-env.cc | 17 -------- be/src/scheduling/query-schedule.cc | 5 --- be/src/service/impala-server.cc | 5 --- be/src/service/impalad-main.cc | 8 ---- 9 files changed, 54 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/common/global-flags.cc ---------------------------------------------------------------------- diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc index 2d832da..8e81d23 100644 --- a/be/src/common/global-flags.cc +++ b/be/src/common/global-flags.cc @@ -160,8 +160,6 @@ DEFINE_int32(kudu_operation_timeout_ms, 3 * 60 * 1000, "Timeout (milliseconds) s "all Kudu operations. This must be a positive value, and there is no way to disable " "timeouts."); -DEFINE_bool_hidden(enable_accept_queue_server, true, "Deprecated"); - DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of " "incremental stats the catalog is allowed to serialize per table. " "This limit is set as a safety check, to prevent the JVM from " @@ -194,3 +192,57 @@ DEFINE_string(reserved_words_version, "3.0.0", "Reserved words compatibility ver "Reserved words cannot be used as identifiers in SQL. This flag determines the impala" " version from which the reserved word list is taken. The value must be one of " "[\"2.11.0\", \"3.0.0\"]."); + +// ++========================++ +// || Startup flag graveyard || +// ++========================++ +// +// ----------- +// -----------/ R I P â² +// / R I P â² -----------|----------- +// |-----------| |/ R I P â² +// | | LLAMA ||-----------| +// | Old Aggs | || | +// | | -- || Old Joins | +// | -- | || | +// | | || -- | +// | |~.~~.~~.~~~~| | +// ~~.~~.~~.~~~~ | | +// ~~.~~.~~.~~~~ +// The flags have no effect but we don't want to prevent Impala from starting when they +// are provided on the command line after an upgrade. We issue a warning if the flag is +// set from the command line. +#define REMOVED_FLAG(flagname) \ + DEFINE_string_hidden(flagname, "__UNSET__", "Removed"); \ + DEFINE_validator(flagname, [](const char* name, const string& val) { \ + if (val != "__UNSET__") LOG(WARNING) << "Ignoring removed flag " << name; \ + return true; \ + }); + +REMOVED_FLAG(be_service_threads); +REMOVED_FLAG(cgroup_hierarchy_path); +REMOVED_FLAG(enable_accept_queue_server); +REMOVED_FLAG(enable_partitioned_aggregation); +REMOVED_FLAG(enable_partitioned_hash_join); +REMOVED_FLAG(enable_phj_probe_side_filtering); +REMOVED_FLAG(enable_rm); +REMOVED_FLAG(llama_addresses); +REMOVED_FLAG(llama_callback_port); +REMOVED_FLAG(llama_host); +REMOVED_FLAG(llama_max_request_attempts); +REMOVED_FLAG(llama_port); +REMOVED_FLAG(llama_registration_timeout_secs); +REMOVED_FLAG(llama_registration_wait_secs); +REMOVED_FLAG(local_nodemanager_url); +REMOVED_FLAG(resource_broker_cnxn_attempts); +REMOVED_FLAG(resource_broker_cnxn_retry_interval_ms); +REMOVED_FLAG(resource_broker_recv_timeout); +REMOVED_FLAG(resource_broker_send_timeout); +REMOVED_FLAG(rm_always_use_defaults); +REMOVED_FLAG(rm_default_cpu_vcores); +REMOVED_FLAG(rm_default_memory); +REMOVED_FLAG(rpc_cnxn_attempts); +REMOVED_FLAG(rpc_cnxn_retry_interval_ms); +REMOVED_FLAG(staging_cgroup); +REMOVED_FLAG(suppress_unknown_disk_id_warnings); +REMOVED_FLAG(use_statestore); http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/exec/exec-node.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc index eaf8dd1..08856f1 100644 --- a/be/src/exec/exec-node.cc +++ b/be/src/exec/exec-node.cc @@ -70,8 +70,6 @@ using strings::Substitute; DECLARE_int32(be_port); DECLARE_string(hostname); -DEFINE_bool_hidden(enable_partitioned_hash_join, true, "Deprecated - has no effect"); -DEFINE_bool_hidden(enable_partitioned_aggregation, true, "Deprecated - has no effect"); namespace impala { http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/exec/hdfs-scan-node-base.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc index 8b065fa..b40fb7e 100644 --- a/be/src/exec/hdfs-scan-node-base.cc +++ b/be/src/exec/hdfs-scan-node-base.cc @@ -46,9 +46,6 @@ #include "common/names.h" -// TODO: Remove this flag in a compatibility-breaking release. -DEFINE_bool(suppress_unknown_disk_id_warnings, false, "Deprecated."); - #ifndef NDEBUG DECLARE_bool(skip_file_runtime_filtering); #endif http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/exec/partitioned-hash-join-node.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/partitioned-hash-join-node.cc b/be/src/exec/partitioned-hash-join-node.cc index e76a9ba..dd3efd1 100644 --- a/be/src/exec/partitioned-hash-join-node.cc +++ b/be/src/exec/partitioned-hash-join-node.cc @@ -39,8 +39,6 @@ #include "common/names.h" -DEFINE_bool_hidden(enable_phj_probe_side_filtering, true, "Deprecated."); - static const string PREPARE_FOR_READ_FAILED_ERROR_MSG = "Failed to acquire initial read buffer for stream in hash join node $0. Reducing " "query concurrency or increasing the memory limit may help this query to complete " http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/rpc/thrift-server.cc ---------------------------------------------------------------------- diff --git a/be/src/rpc/thrift-server.cc b/be/src/rpc/thrift-server.cc index 48fb1b9..8f948cd 100644 --- a/be/src/rpc/thrift-server.cc +++ b/be/src/rpc/thrift-server.cc @@ -58,9 +58,6 @@ using namespace apache::thrift::server; using namespace apache::thrift::transport; using namespace apache::thrift; -DEFINE_int32_hidden(rpc_cnxn_attempts, 10, "Deprecated"); -DEFINE_int32_hidden(rpc_cnxn_retry_interval_ms, 2000, "Deprecated"); - DECLARE_string(principal); DECLARE_string(keytab_file); http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/runtime/exec-env.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc index 1c3ab7a..ff6a374 100644 --- a/be/src/runtime/exec-env.cc +++ b/be/src/runtime/exec-env.cc @@ -70,7 +70,6 @@ using boost::algorithm::join; using kudu::rpc::ServiceIf; using namespace strings; -DEFINE_bool_hidden(use_statestore, true, "Deprecated, do not use"); DEFINE_string(catalog_service_host, "localhost", "hostname where CatalogService is running"); DEFINE_bool(enable_webserver, true, "If true, debug webserver is enabled"); @@ -103,22 +102,6 @@ DECLARE_bool(is_coordinator); DECLARE_int32(webserver_port); DECLARE_int64(tcmalloc_max_total_thread_cache_bytes); -// TODO: Remove the following RM-related flags in Impala 3.0. -DEFINE_bool_hidden(enable_rm, false, "Deprecated"); -DEFINE_int32_hidden(llama_callback_port, 28000, "Deprecated"); -DEFINE_string_hidden(llama_host, "", "Deprecated"); -DEFINE_int32_hidden(llama_port, 15000, "Deprecated"); -DEFINE_string_hidden(llama_addresses, "", "Deprecated"); -DEFINE_int64_hidden(llama_registration_timeout_secs, 30, "Deprecated"); -DEFINE_int64_hidden(llama_registration_wait_secs, 3, "Deprecated"); -DEFINE_int64_hidden(llama_max_request_attempts, 5, "Deprecated"); -DEFINE_string_hidden(cgroup_hierarchy_path, "", "Deprecated"); -DEFINE_string_hidden(staging_cgroup, "impala_staging", "Deprecated"); -DEFINE_int32_hidden(resource_broker_cnxn_attempts, 1, "Deprecated"); -DEFINE_int32_hidden(resource_broker_cnxn_retry_interval_ms, 3000, "Deprecated"); -DEFINE_int32_hidden(resource_broker_send_timeout, 0, "Deprecated"); -DEFINE_int32_hidden(resource_broker_recv_timeout, 0, "Deprecated"); - // TODO-MT: rename or retire DEFINE_int32(coordinator_rpc_threads, 12, "(Advanced) Number of threads available to " "start fragments on remote Impala daemons."); http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/scheduling/query-schedule.cc ---------------------------------------------------------------------- diff --git a/be/src/scheduling/query-schedule.cc b/be/src/scheduling/query-schedule.cc index f833273..a424f78 100644 --- a/be/src/scheduling/query-schedule.cc +++ b/be/src/scheduling/query-schedule.cc @@ -35,11 +35,6 @@ using boost::uuids::random_generator; using boost::uuids::uuid; using namespace impala; -// TODO: Remove for Impala 3.0. -DEFINE_bool_hidden(rm_always_use_defaults, false, "Deprecated"); -DEFINE_string_hidden(rm_default_memory, "4G", "Deprecated"); -DEFINE_int32_hidden(rm_default_cpu_vcores, 2, "Deprecated"); - namespace impala { QuerySchedule::QuerySchedule(const TUniqueId& query_id, http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/service/impala-server.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index ee8405b..9898029 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -118,8 +118,6 @@ DEFINE_int32(hs2_port, 21050, "port on which HiveServer2 client requests are ser DEFINE_int32(fe_service_threads, 64, "number of threads available to serve client requests"); -DEFINE_int32_hidden(be_service_threads, 64, - "Deprecated, no longer has any effect. Will be removed in Impala 3.0."); DEFINE_string(default_query_options, "", "key=value pair of default query options for" " impalad, separated by ','"); DEFINE_int32(query_log_size, 25, "Number of queries to retain in the query log. If -1, " @@ -208,9 +206,6 @@ DEFINE_bool(is_executor, true, "If true, this Impala daemon will execute query " "milliseconds. Only used for testing."); #endif -// TODO: Remove for Impala 3.0. -DEFINE_string_hidden(local_nodemanager_url, "", "Deprecated"); - DECLARE_bool(compact_catalog_topic); namespace impala { http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/service/impalad-main.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impalad-main.cc b/be/src/service/impalad-main.cc index e0049d0..8463467 100644 --- a/be/src/service/impalad-main.cc +++ b/be/src/service/impalad-main.cc @@ -54,7 +54,6 @@ using namespace impala; DECLARE_int32(beeswax_port); DECLARE_int32(hs2_port); DECLARE_int32(be_port); -DECLARE_bool(enable_rm); DECLARE_bool(is_coordinator); int ImpaladMain(int argc, char** argv) { @@ -70,13 +69,6 @@ int ImpaladMain(int argc, char** argv) { ABORT_IF_ERROR(JniCatalogCacheUpdateIterator::InitJNI()); InitFeSupport(); - if (FLAGS_enable_rm) { - // TODO: Remove in Impala 3.0. - LOG(WARNING) << "*****************************************************************"; - LOG(WARNING) << "Llama support has been deprecated. FLAGS_enable_rm has no effect."; - LOG(WARNING) << "*****************************************************************"; - } - ExecEnv exec_env; ABORT_IF_ERROR(exec_env.Init()); CommonMetrics::InitCommonMetrics(exec_env.metrics());
