Repository: incubator-impala Updated Branches: refs/heads/master 3f5380dc7 -> ee2a06d82
Remove Llama dependency This change prevents us from depending on LLAMA to build. Note that the LLAMA MiniKDC is left in - it is a test utility that does not depend on LLAMA itself. IMPALA-4292 tracks cleaning this up. Testing: Ran a private build to verify that all tests pass. Change-Id: If2e5e21d8047097d56062ded11b0832a1d397fe0 Reviewed-on: http://gerrit.cloudera.org:8080/4739 Reviewed-by: Tim Armstrong <tarmstr...@cloudera.com> Reviewed-by: Henry Robinson <he...@cloudera.com> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ee2a06d8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ee2a06d8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ee2a06d8 Branch: refs/heads/master Commit: ee2a06d827a94e659e7d92fe027eeaab6a9ac680 Parents: 3f5380d Author: Tim Armstrong <tarmstr...@cloudera.com> Authored: Thu Oct 13 13:24:43 2016 -0700 Committer: Tim Armstrong <tarmstr...@cloudera.com> Committed: Tue Oct 18 16:35:58 2016 +0000 ---------------------------------------------------------------------- bin/bootstrap_toolchain.py | 2 +- bin/generate_minidump_collection_testdata.py | 1 - bin/impala-config.sh | 2 -- bin/start-impala-cluster.py | 7 ++----- common/thrift/ImpalaInternalService.thrift | 6 ++++++ common/thrift/ImpalaService.thrift | 5 ++++- common/thrift/generate_metrics.py | 4 ---- common/thrift/metrics.json | 20 -------------------- infra/deploy/deploy.py | 9 --------- 9 files changed, 13 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/bootstrap_toolchain.py ---------------------------------------------------------------------- diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 6524e82..3f161c8 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -351,5 +351,5 @@ if __name__ == "__main__": # Download the CDH components if necessary. if os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true": - cdh_components = ["hadoop", "hbase", "hive", "llama", "llama-minikdc", "sentry"] + cdh_components = ["hadoop", "hbase", "hive", "llama-minikdc", "sentry"] download_cdh_components(toolchain_root, cdh_components) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/generate_minidump_collection_testdata.py ---------------------------------------------------------------------- diff --git a/bin/generate_minidump_collection_testdata.py b/bin/generate_minidump_collection_testdata.py index a408e05..350f0e3 100755 --- a/bin/generate_minidump_collection_testdata.py +++ b/bin/generate_minidump_collection_testdata.py @@ -50,7 +50,6 @@ options, args = parser.parse_args() CONFIG_FILE = '''-beeswax_port=21000 -fe_port=21000 -be_port=22000 --llama_callback_port=28000 -hs2_port=21050 -enable_webserver=true -mem_limit=108232130560 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 2a25248..38bdc69 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -299,7 +299,6 @@ export IMPALA_HADOOP_VERSION=${IMPALA_HADOOP_VERSION:-2.6.0-cdh5.10.0-SNAPSHOT} export IMPALA_HBASE_VERSION=${IMPALA_HBASE_VERSION:-1.2.0-cdh5.10.0-SNAPSHOT} export IMPALA_HIVE_VERSION=${IMPALA_HIVE_VERSION:-1.1.0-cdh5.10.0-SNAPSHOT} export IMPALA_SENTRY_VERSION=${IMPALA_SENTRY_VERSION:-1.5.1-cdh5.10.0-SNAPSHOT} -export IMPALA_LLAMA_VERSION=${IMPALA_LLAMA_VERSION:-1.0.0-cdh5.10.0-SNAPSHOT} export IMPALA_PARQUET_VERSION=${IMPALA_PARQUET_VERSION:-1.5.0-cdh5.10.0-SNAPSHOT} export IMPALA_LLAMA_MINIKDC_VERSION=${IMPALA_LLAMA_MINIKDC_VERSION:-1.0.0} @@ -339,7 +338,6 @@ HADOOP_CLASSPATH+=":$LZO_JAR_PATH" export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data" export PATH="$HADOOP_HOME/bin:$PATH" -export LLAMA_HOME="$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/" export MINIKDC_HOME="$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}" export SENTRY_HOME="$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}" export SENTRY_CONF_DIR="$IMPALA_HOME/fe/src/test/resources" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/start-impala-cluster.py ---------------------------------------------------------------------- diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py index b92fcf1..3ea338a 100755 --- a/bin/start-impala-cluster.py +++ b/bin/start-impala-cluster.py @@ -85,8 +85,7 @@ MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process" IMPALA_SHELL = os.path.join(IMPALA_HOME, 'bin/impala-shell.sh') IMPALAD_PORTS = ("-beeswax_port=%d -hs2_port=%d -be_port=%d " - "-state_store_subscriber_port=%d -webserver_port=%d " - "-llama_callback_port=%d") + "-state_store_subscriber_port=%d -webserver_port=%d") JVM_ARGS = "-jvm_debug_port=%s -jvm_args=%s" BE_LOGGING_ARGS = "-log_filename=%s -log_dir=%s -v=%s -logbufsecs=5 -max_log_files=%s" CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240 @@ -188,12 +187,10 @@ def build_impalad_port_args(instance_num): BASE_BE_PORT = 22000 BASE_STATE_STORE_SUBSCRIBER_PORT = 23000 BASE_WEBSERVER_PORT = 25000 - BASE_LLAMA_CALLBACK_PORT = 28000 return IMPALAD_PORTS % (BASE_BEESWAX_PORT + instance_num, BASE_HS2_PORT + instance_num, BASE_BE_PORT + instance_num, BASE_STATE_STORE_SUBSCRIBER_PORT + instance_num, - BASE_WEBSERVER_PORT + instance_num, - BASE_LLAMA_CALLBACK_PORT + instance_num) + BASE_WEBSERVER_PORT + instance_num) def build_impalad_logging_args(instance_num, service_name): log_file_path = os.path.join(options.log_dir, "%s.INFO" % service_name) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/ImpalaInternalService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift index 3ee54ae..736de34 100644 --- a/common/thrift/ImpalaInternalService.thrift +++ b/common/thrift/ImpalaInternalService.thrift @@ -71,11 +71,14 @@ struct TQueryOptions { 6: optional i64 max_scan_range_length = 0 7: optional i32 num_scanner_threads = 0 + // TODO: IMPALA-4306: retire at compatibility-breaking version 8: optional i32 max_io_buffers = 0 // Deprecated in 1.1 9: optional bool allow_unsupported_formats = 0 + // TODO: IMPALA-4306: retire at compatibility-breaking version 10: optional i64 default_order_by_limit = -1 // Deprecated in 1.4 11: optional string debug_action = "" 12: optional i64 mem_limit = 0 + // TODO: IMPALA-4306: retire at compatibility-breaking version 13: optional bool abort_on_default_limit_exceeded = 0 // Deprecated in 1.4 14: optional CatalogObjects.THdfsCompression compression_codec 15: optional i32 hbase_caching = 0 @@ -89,10 +92,12 @@ struct TQueryOptions { 20: optional string request_pool // Per-host virtual CPU cores required for query (only relevant with RM). + // TODO: IMPALA-3271: retire at compatibility-breaking version 21: optional i16 v_cpu_cores // Max time in milliseconds the resource broker should wait for // a resource request to be granted by Llama/Yarn (only relevant with RM). + // TODO: IMPALA-3271: retire at compatibility-breaking version 22: optional i64 reservation_request_timeout // Disables taking advantage of HDFS caching. This has two parts: @@ -104,6 +109,7 @@ struct TQueryOptions { 24: optional bool disable_outermost_topn = 0 // Override for initial memory reservation size if RM is enabled. + // TODO: IMPALA-3271: retire at compatibility-breaking version 25: optional i64 rm_initial_mem = 0 // Time, in s, before a query will be timed out if it is inactive. May not exceed http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/ImpalaService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift index 129be2d..794c140 100644 --- a/common/thrift/ImpalaService.thrift +++ b/common/thrift/ImpalaService.thrift @@ -134,21 +134,24 @@ enum TImpalaQueryOptions { REQUEST_POOL, // Per-host virtual CPU cores required for query (only relevant with RM). + // TODO: IMPALA-3271: retire at compatibility-breaking version V_CPU_CORES, // Max time in milliseconds the resource broker should wait for // a resource request to be granted by Llama/Yarn (only relevant with RM). + // TODO: IMPALA-3271: retire at compatibility-breaking version RESERVATION_REQUEST_TIMEOUT, // if true, disables cached reads. This option has no effect if REPLICA_PREFERENCE is // configured. - // TODO: Retire in C6 + // TODO: IMPALA-4306: retire at compatibility-breaking version DISABLE_CACHED_READS, // Temporary testing flag DISABLE_OUTERMOST_TOPN, // Size of initial memory reservation when RM is enabled + // TODO: IMPALA-3271: retire at compatibility-breaking version RM_INITIAL_MEM, // Time, in s, before a query will be timed out if it is inactive. May not exceed http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/generate_metrics.py ---------------------------------------------------------------------- diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py index e878a29..f1f57cd 100755 --- a/common/thrift/generate_metrics.py +++ b/common/thrift/generate_metrics.py @@ -166,10 +166,6 @@ MDL_BASE = """ "name" : "CATALOGSERVER", "nameForCrossEntityAggregateMetrics" : "catalogservers" }, - { - "name" : "LLAMA", - "nameForCrossEntityAggregateMetrics" : "llamas" - } ], "metricEntityTypeDefinitions" : [ { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/metrics.json ---------------------------------------------------------------------- diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json index c61c7fe..0829139 100644 --- a/common/thrift/metrics.json +++ b/common/thrift/metrics.json @@ -570,26 +570,6 @@ "key": "impala-server.version" }, { - "description": "The number of active connections to the Llama Callback Service.", - "contexts": [ - "IMPALAD" - ], - "label": "Llama Callback Service Active Connections", - "units": "NONE", - "kind": "GAUGE", - "key": "impala.thrift-server.llama-callback.connections-in-use" - }, - { - "description": "The total number of connections made to the Llama Callback Service over its lifetime.", - "contexts": [ - "IMPALAD" - ], - "label": "Llama Callback Service Total Connections", - "units": "UNIT", - "kind": "COUNTER", - "key": "impala.thrift-server.llama-callback.total-connections" - }, - { "description": "The number of active catalog service connections to this Catalog Server.", "contexts": [ "CATALOGSERVER" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/infra/deploy/deploy.py ---------------------------------------------------------------------- diff --git a/infra/deploy/deploy.py b/infra/deploy/deploy.py index 56e3c46..0463111 100644 --- a/infra/deploy/deploy.py +++ b/infra/deploy/deploy.py @@ -314,8 +314,6 @@ def create_new_service(api, cluster, new_name, deps, scratch_dirs, master_host): new_role_name = "%s-%s-%s" % (new_name, rcg.roleType, md5.hexdigest()) new_service.create_role(new_role_name, rcg.roleType, h.hostId) print "Created new role %s" % (new_role_name,) - elif rcg.roleType == "LLAMA": - continue else: md5 = hashlib.md5() md5.update(master_host.hostId) @@ -367,19 +365,12 @@ def transform_rcg_config(rcg): new_config.update(transform_port(rcg.name, full, "beeswax_port")) new_config.update(transform_port(rcg.name, full, "hs2_port")) new_config.update(transform_port(rcg.name, full, "impalad_webserver_port")) - new_config.update(transform_port(rcg.name, full, "llama_callback_port")) new_config.update(transform_port(rcg.name, full, "state_store_subscriber_port")) elif rcg.roleType == "CATALOGSERVER": new_config.update(transform_path(rcg.name, full, "log_dir")) new_config.update(transform_port(rcg.name, full, "catalog_service_port")) new_config.update(transform_port(rcg.name, full, "catalogserver_webserver_port")) - elif rcg.roleType == "LLAMA": - new_config.update(transform_path(rcg.name, full, "llama_log_dir")) - - new_config.update(transform_port(rcg.name, full, "llama_am_server_thrift_admin_address")) - new_config.update(transform_port(rcg.name, full, "llama_http_port")) - new_config.update(transform_port(rcg.name, full, "llama_port")) elif rcg.roleType == "STATESTORE": new_config.update(transform_path(rcg.name, full, "log_dir"))
