http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/common/thrift/ResourceBrokerService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ResourceBrokerService.thrift b/common/thrift/ResourceBrokerService.thrift deleted file mode 100644 index 36f1db8..0000000 --- a/common/thrift/ResourceBrokerService.thrift +++ /dev/null @@ -1,119 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace cpp impala -namespace java com.cloudera.impala.thrift - -include "Status.thrift" -include "Types.thrift" -include "Llama.thrift" - -enum TResourceBrokerServiceVersion { - V1 -} - -struct TResourceBrokerRegisterRequest { - 1: required TResourceBrokerServiceVersion version; - 2: optional i32 client_process_id; - 3: optional Types.TNetworkAddress notification_callback_service; -} - -struct TResourceBrokerRegisterResponse { - 1: optional Status.TStatus status; - 2: optional Types.TUniqueId irb_handle; -} - -struct TResourceBrokerUnregisterRequest { - 1: optional TResourceBrokerServiceVersion version; - 2: optional Types.TUniqueId irb_handle; -} - -struct TResourceBrokerUnregisterResponse { - 1: optional Status.TStatus status; -} - -struct TResourceBrokerExpansionRequest { - 1: required TResourceBrokerServiceVersion version; - 2: optional Llama.TResource resource; - 3: optional Types.TUniqueId reservation_id; - 4: optional i64 request_timeout; -} - -struct TResourceBrokerExpansionResponse { - 1: optional Types.TUniqueId reservation_id; - 2: optional map<Types.TNetworkAddress, Llama.TAllocatedResource> allocated_resources; -} - -struct TResourceBrokerReservationRequest { - 1: required TResourceBrokerServiceVersion version; - 2: optional Types.TUniqueId irb_handle; - 3: optional string queue; - 4: optional list<Llama.TResource> resources; - - // If true, requires a reservation response to either grant or deny all resources - // in this request. If false, reservation responses may deliver partially - // granted/denied resources. - 5: optional bool gang; - - // Max time in milliseconds the resource broker should wait for - // a resource request to be granted by Llama/Yarn. - 6: optional i64 request_timeout; - - // Used to allow Llama to grant or deny access to the requested queue. - 7: optional string user; -} - -struct TResourceBrokerReservationResponse { - 1: optional Status.TStatus status; - 2: optional Types.TUniqueId reservation_id; - 3: optional map<Types.TNetworkAddress, Llama.TAllocatedResource> allocated_resources; -} - -struct TResourceBrokerReleaseRequest { - 1: optional TResourceBrokerServiceVersion version; - 2: optional Types.TUniqueId irb_handle; - 3: optional Types.TUniqueId reservation_id; -} - -struct TResourceBrokerReleaseResponse { - 1: optional Status.TStatus status; -} - -service ResourceBrokerService { - TResourceBrokerRegisterResponse - Register(1: TResourceBrokerRegisterRequest request); - TResourceBrokerUnregisterResponse - Unregister(1: TResourceBrokerUnregisterRequest request); - TResourceBrokerReservationResponse - Reserve(1: TResourceBrokerReservationRequest request); - TResourceBrokerReleaseResponse - Release(1: TResourceBrokerReleaseRequest request); -} - -struct TResourceBrokerNotificationPreemptionRequest { - 1: required TResourceBrokerServiceVersion version; - 2: optional list<string> preempted_rm_resource_ids; -} - -struct TResourceBrokerNotificationPreemptionResponse { - 1: optional Status.TStatus status; -} - -service ResourceBrokerNotificationService { - TResourceBrokerRegisterResponse - Preempt(1: TResourceBrokerRegisterRequest request); -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/common/thrift/metrics.json ---------------------------------------------------------------------- diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json index 1d449b5..b600f80 100644 --- a/common/thrift/metrics.json +++ b/common/thrift/metrics.json @@ -740,216 +740,6 @@ "key": "request-pool-service.resolve-pool-duration-ms" }, { - "description": "The number of clients in use by the resource broker client cache.", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Client Cache Clients In Use", - "units": "NONE", - "kind": "GAUGE", - "key": "resource-broker.client-cache.clients-in-use" - }, - { - "description": "The total number of clients in the resource broker client cache.", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Client Cache Total Clients", - "units": "NONE", - "kind": "GAUGE", - "key": "resource-broker.client-cache.total-clients" - }, - { - "key": "resource-broker.active-llama", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Active Llama", - "units": "NONE", - "kind": "PROPERTY", - "description": "The address of the active Llama server." - }, - { - "key": "resource-broker.active-llama-handle", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Active Llama Handle", - "units": "NONE", - "kind": "PROPERTY", - "description": "The Llama client handle provided by the active Llama server." - }, - { - "key": "resource-broker.expansion-request-response-time", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Request Response Time", - "units": "TIME_S", - "kind": "STATS", - "description": "The time, in seconds, that a expansion request takes to be fulfilled by Llama" - }, - { - "key": "resource-broker.expansion-request-rpc-time", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Request Rpc Time", - "units": "TIME_S", - "kind": "STATS", - "description": "The time, in seconds, that a Reserve() RPC takes to Llama" - }, - { - "key": "resource-broker.expansion-requests-failed", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Requests Failed", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of expansion requests made by this Impala daemon to Llama which failed" - }, - { - "key": "resource-broker.expansion-requests-fulfilled", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Requests Fulfilled", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of expansion requests made by this Impala daemon to Llama which succeeded" - }, - { - "key": "resource-broker.expansion-requests-rejected", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Requests Rejected", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of expansion requests made by this Impala daemon to Llama which were rejected" - }, - { - "key": "resource-broker.expansion-requests-timedout", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Requests Timedout", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of expansion requests made by this Impala daemon to Llama which timed out" - }, - { - "key": "resource-broker.expansion-requests-total", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Expansion Requests Total", - "units": "UNIT", - "kind": "COUNTER", - "description": "The total number of expansion requests made by this Impala daemon to Llama" - }, - { - "key": "resource-broker.memory-resources-in-use", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Memory Resources In Use", - "units": "BYTES", - "kind": "GAUGE", - "description": "The total number of bytes currently allocated to this Impala daemon by Llama" - }, - { - "key": "resource-broker.requests-released", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Requests Released", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of resource-release requests received from Llama" - }, - { - "key": "resource-broker.reservation-request-response-time", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Request Response Time", - "units": "TIME_S", - "kind": "STATS", - "description": "The time, in seconds, that a reservation request takes to be fulfilled by Llama" - }, - { - "key": "resource-broker.reservation-request-rpc-time", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Request Rpc Time", - "units": "TIME_S", - "kind": "STATS", - "description": "The time, in seconds, that a Reserve() RPC takes to Llama" - }, - { - "key": "resource-broker.reservation-requests-failed", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Requests Failed", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of reservation requests made by this Impala daemon to Llama which failed" - }, - { - "key": "resource-broker.reservation-requests-fulfilled", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Requests Fulfilled", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of reservation requests made by this Impala daemon to Llama which succeeded" - }, - { - "key": "resource-broker.reservation-requests-rejected", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Requests Rejected", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of reservation requests made by this Impala daemon to Llama which were rejected" - }, - { - "key": "resource-broker.reservation-requests-timedout", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Requests Timedout", - "units": "UNIT", - "kind": "COUNTER", - "description": "The number of reservation requests made by this Impala daemon to Llama which timed out" - }, - { - "key": "resource-broker.reservation-requests-total", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Reservation Requests Total", - "units": "UNIT", - "kind": "COUNTER", - "description": "The total number of reservation requests made by this Impala daemon to Llama" - }, - { - "key": "resource-broker.vcpu-resources-in-use", - "contexts": [ - "IMPALAD" - ], - "label": "Resource Broker Vcpu Resources In Use", - "units": "UNIT", - "kind": "GAUGE", - "description": "The total number of vcpus currently allocated to this Impala daemon by Llama" - }, - { "description": "Duration (ms) of RPC calls to $0", "contexts": [ "CATALOGSERVER", http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/fe/src/main/java/com/cloudera/impala/planner/Planner.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/planner/Planner.java b/fe/src/main/java/com/cloudera/impala/planner/Planner.java index 0a424ad..df90df3 100644 --- a/fe/src/main/java/com/cloudera/impala/planner/Planner.java +++ b/fe/src/main/java/com/cloudera/impala/planner/Planner.java @@ -292,6 +292,7 @@ public class Planner { * Optionally excludes the requirements for unpartitioned fragments. * TODO: The LOG.warn() messages should eventually become Preconditions checks * once resource estimation is more robust. + * TODO: Revisit and possibly remove during MT work, particularly references to vcores. */ public void computeResourceReqs(List<PlanFragment> fragments, boolean excludeUnpartitionedFragments, http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/testdata/cluster/admin ---------------------------------------------------------------------- diff --git a/testdata/cluster/admin b/testdata/cluster/admin index af35009..d263999 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -55,7 +55,7 @@ export KILL_CLUSTER_MARKER=IBelongToTheMiniCluster if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then # The check above indicates that the regular mini-cluster is in use. - SUPPORTED_SERVICES=(hdfs yarn llama kms) + SUPPORTED_SERVICES=(hdfs yarn kms) else # Either a remote distributed file system or a local non-distributed file system is # in use. Currently the only service that is expected to work is Kudu, though in theory @@ -87,9 +87,6 @@ export KUDU_WEBUI_PORT=8051 # same as default # easier to maintain them here. EMPTY_NODE_DIRS=$(echo data/dfs/{dn,nn} var/{run,lib/hadoop-hdfs,log} \ var/{log,run}/kudu/{master,ts} var/lib/kudu/{master,ts}/{wal,data}) -if [[ "$CDH_MAJOR_VERSION" -ge 5 ]]; then - EMPTY_NODE_DIRS+=" /var/log/llama" -fi EASY_ACCESS_LOG_DIR="$IMPALA_CLUSTER_LOGS_DIR" MINIKDC_INIT=${IMPALA_HOME}/testdata/bin/minikdc.sh @@ -226,10 +223,7 @@ function create_cluster { fi fi - # For consistency, the first node will host all the master roles, including llama. Llama - # needs to know the ports of the datanodes and nodemanagers. The ports are generated - # below as part of setting up each node. The slave nodes are created first so the ports - # will be known when it is time to configure llama. + # For consistency, the first node will host all the master roles. for ((NODE_IDX=$NODE_COUNT; NODE_IDX >= 1; NODE_IDX--)); do NODE=${NODE_PREFIX}$NODE_IDX NODE_DIR=$(get_node_dir $NODE) @@ -244,7 +238,7 @@ function create_cluster { if [[ $NODE_IDX -gt 1 ]]; then # Remove master role scripts from slave nodes rm -f "$NODE_DIR/etc/init.d/"{hdfs-namenode,yarn-resourcemanager} \ - "$NODE_DIR/etc/init.d/"{llama-application,kms,kudu-master} + "$NODE_DIR/etc/init.d/"{kms,kudu-master} fi for EMPTY_NODE_DIR in $EMPTY_NODE_DIRS; do mkdir -p "$NODE_DIR/$EMPTY_NODE_DIR" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl index 0d4b882..3a01d75 100644 --- a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl +++ b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl @@ -1,24 +1,13 @@ <?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- TODO: Remove any Llama-related configuration. Can this file be removed entirely? --> <configuration> <property> <name>yarn.resourcemanager.webapp.address</name> <value>0.0.0.0:${YARN_WEBUI_PORT}</value> </property> - <!-- Min value of 0 is required by llama --> - <property> - <name>yarn.scheduler.minimum-allocation-mb</name> - <value>0</value> - </property> - - <!-- Min value of 0 is required by llama --> - <property> - <name>yarn.scheduler.minimum-allocation-vcores</name> - <value>0</value> - </property> - <property> <name>yarn.nodemanager.address</name> <value>127.0.0.1:${NODEMANAGER_PORT}</value> http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/testdata/cluster/node_templates/cdh5/etc/init.d/llama-application ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/init.d/llama-application b/testdata/cluster/node_templates/cdh5/etc/init.d/llama-application deleted file mode 100755 index ad24240..0000000 --- a/testdata/cluster/node_templates/cdh5/etc/init.d/llama-application +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -DIR=$(dirname $0) - -. "$DIR/common" -. "$DIR/hdfs-common" - -HADOOP_LOG_DIR="$LOG_DIR/llama" -HADOOP_CLASSPATH="$NODE_DIR/etc/llama/conf:$HADOOP_CLASSPATH" -for JAR in $(find "$LLAMA_HOME" -name "*jar"); do - HADOOP_CLASSPATH="$JAR:$HADOOP_CLASSPATH" -done -export HADOOP_CLASSPATH -export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS \ - -Dllama.server.conf.dir=$NODE_DIR/etc/llama/conf" - -function start { - do_start hadoop 'com.cloudera.llama.am.LlamaAMMain' -} - -$1 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-log4j.properties.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-log4j.properties.tmpl b/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-log4j.properties.tmpl deleted file mode 100644 index ba62c0f..0000000 --- a/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-log4j.properties.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -log4j.appender.RFA=org.apache.log4j.RollingFileAppender -log4j.appender.RFA.File=${NODE_DIR}/var/log/llama/llama-application.log -log4j.appender.RFA.MaxFileSize=256MB -log4j.appender.RFA.MaxBackupIndex=20 -log4j.appender.RFA.layout=org.apache.log4j.PatternLayout -log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -log4j.logger.org.apache.hadoop.conf=ERROR, RFA -log4j.logger.org.apache.hadoop=WARN, RFA -log4j.logger.com.cloudera.llama=INFO, RFA -#log4j.logger.org.apache.thrift.transport=DEBUG, RFA http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-site.xml.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-site.xml.tmpl b/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-site.xml.tmpl deleted file mode 100644 index 90232c0..0000000 --- a/testdata/cluster/node_templates/cdh5/etc/llama/conf/llama-site.xml.tmpl +++ /dev/null @@ -1,86 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> - <property> - <name>llama.am.server.thrift.http.address</name> - <value>0.0.0.0:${LLAMA_WEBUI_PORT}</value> - </property> - - <!-- This is causing problems now, re-enable it later. --> - <property> - <name>llama.am.cache.enabled</name> - <value>false</value> - <description> - Global setting that indicates if Llama should cache allocated resources on - release. - </description> - </property> - - <!-- This is a yarn config but Llama needs it here. --> - <property> - <name>yarn.scheduler.include-port-in-node-name</name> - <value>true</value> - </property> - - <property> - <name>llama.am.server.thrift.node.name.mapping.class</name> - <value>com.cloudera.llama.am.MiniClusterNodeMapper</value> - <description> - The NodeMapper implementation LlamaAM server uses to convert requested - locations into Yarn Nodes. - The default (and only implementation for production) drops the port - number if present (Impala uses DataNode addresses to request a location, - these addresses may contain the DataNode port number. The DataNode port - number is meaningless and unknown to Yarn). - </description> - </property> - - <property> - <name>llama.minicluster.node.mapper.mapping</name> - <value>${LLAMA_PORT_MAPPINGS}</value> - </property> - - <property> - <name>llama.am.hadoop.user.name</name> - <value>${USER}</value> - <description> - User name use by Llama when interacting with Yarn. - </description> - </property> - -<!-- BEGIN Kerberos settings --> - <property> - <name>llama.am.server.thrift.security</name> - <value>true</value> - </property> - - <property> - <name>llama.am.server.thrift.kerberos.keytab.file</name> - <value>${KRB5_KTNAME}</value> - </property> - - <property> - <name>llama.am.server.thrift.kerberos.server.principal.name</name> - <value>${MINIKDC_PRINC_LLAM}</value> - </property> - - <property> - <name>llama.am.server.thrift.kerberos.notification.principal.name</name> - <!-- <value>${MINIKDC_PRINC_IMPALA_BE}</value> --> - <value>impala-be</value> - </property> -<!-- END Kerberos settings --> - -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/19de09ab/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl index 2d5cb09..4f4af77 100644 --- a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl +++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl @@ -110,16 +110,6 @@ DEFAULT</value> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.llama.hosts</name> - <value>*</value> - </property> - - <property> - <name>hadoop.proxyuser.llama.groups</name> - <value>*</value> - </property> - <!-- END Kerberos settings --> </configuration>