Repository: hive Updated Branches: refs/heads/llap cb24a7678 -> f032e5488
HIVE-9814 : LLAP: JMX web-service end points for monitoring & metrics (Gopal V, Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/54127078 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/54127078 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/54127078 Branch: refs/heads/llap Commit: 54127078f3a87b9f8dc9529467a04206be0cc398 Parents: cb24a76 Author: Sergey Shelukhin <[email protected]> Authored: Wed Jun 3 14:25:38 2015 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Wed Jun 3 14:25:38 2015 -0700 ---------------------------------------------------------------------- .../llap/configuration/LlapConfiguration.java | 6 +++ llap-server/pom.xml | 11 ++++ .../hive/llap/daemon/impl/LlapDaemon.java | 6 +++ .../impl/LlapDaemonProtocolServerImpl.java | 2 +- .../registry/impl/LlapRegistryService.java | 18 ------- .../registry/impl/LlapYarnRegistryImpl.java | 24 +++++++++ .../llap/daemon/services/impl/LlapWebApp.java | 12 +++++ .../daemon/services/impl/LlapWebServices.java | 53 ++++++++++++++++++++ .../src/main/resources/webapps/llap/.keep | 0 9 files changed, 113 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java ---------------------------------------------------------------------- diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java index 194deee..b57ae80 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java @@ -133,4 +133,10 @@ public class LlapConfiguration extends Configuration { public static final String LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MILLIS = LLAP_PREFIX + "task.communicator.connection.sleep-between-retries-millis"; public static final long LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MILLIS_DEFAULT = 2000l; + + public static final String LLAP_DAEMON_SERVICE_PORT = LLAP_DAEMON_PREFIX + "service.port"; + public static final int LLAP_DAEMON_SERVICE_PORT_DEFAULT = 15002; + + public static final String LLAP_DAEMON_SERVICE_SSL = LLAP_DAEMON_PREFIX + "service.ssl"; + public static final boolean LLAP_DAEMON_SERVICE_SSL_DEFAULT = false; } http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/pom.xml ---------------------------------------------------------------------- diff --git a/llap-server/pom.xml b/llap-server/pom.xml index 4fcd705..19f41fe 100644 --- a/llap-server/pom.xml +++ b/llap-server/pom.xml @@ -235,6 +235,17 @@ <build> <sourceDirectory>${basedir}/src/java</sourceDirectory> <testSourceDirectory>${basedir}/src/test</testSourceDirectory> + <resources> + <resource> + <directory>src/main/resources</directory> + <excludes> + <exclude>*.py</exclude> + <exclude>*.pyc</exclude> + <exclude>llap-daemon-log4j.properties</exclude> + </excludes> + <filtering>false</filtering> + </resource> + </resources> <plugins> <plugin> <groupId>org.codehaus.mojo</groupId> http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index d55b7cf..067e213 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryComp import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto; +import org.apache.hadoop.hive.llap.daemon.services.impl.LlapWebServices; import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics; import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem; @@ -54,6 +55,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap private final LlapDaemonProtocolServerImpl server; private final ContainerRunnerImpl containerRunner; private final LlapRegistryService registry; + private final LlapWebServices webServices; private final AtomicLong numSubmissions = new AtomicLong(0); private JvmPauseMonitor pauseMonitor; private final ObjectName llapDaemonInfoBean; @@ -153,6 +155,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap metrics); this.registry = new LlapRegistryService(true); + this.webServices = new LlapWebServices(); } private void printAsciiArt() { @@ -174,6 +177,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap server.init(conf); containerRunner.init(conf); registry.init(conf); + webServices.init(conf); LlapIoProxy.setDaemon(true); LlapIoProxy.initializeLlapIo(conf); } @@ -185,6 +189,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap containerRunner.start(); registry.start(); registry.registerWorker(); + webServices.start(); } public void serviceStop() throws Exception { @@ -194,6 +199,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap server.stop(); registry.unregisterWorker(); registry.stop(); + webServices.stop(); ShuffleHandler.shutdown(); } http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java index 8cb9715..cf3cc78 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java @@ -68,7 +68,7 @@ public class LlapDaemonProtocolServerImpl extends AbstractService } @Override - public SubmitWorkResponseProto submitWork(RpcController controller, + public SubmitWorkResponseProto submitWork (RpcController controller, SubmitWorkRequestProto request) throws ServiceException { try { http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java index 9f5702d..f727e09 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java @@ -14,31 +14,13 @@ package org.apache.hadoop.hive.llap.daemon.registry.impl; import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.Map; -import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.registry.ServiceInstanceSet; import org.apache.hadoop.hive.llap.daemon.registry.ServiceRegistry; -import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal; -import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ProtocolTypes; -import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.log4j.Logger; -import org.apache.zookeeper.CreateMode; - -import com.google.common.base.Preconditions; public class LlapRegistryService extends AbstractService { http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java index ace080e..cb1b1d0 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java @@ -16,6 +16,9 @@ package org.apache.hadoop.hive.llap.daemon.registry.impl; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; import java.net.UnknownHostException; import java.util.HashMap; import java.util.HashSet; @@ -46,6 +49,7 @@ import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.log4j.Logger; +import org.apache.tez.dag.api.TezUncheckedException; import org.apache.zookeeper.CreateMode; import com.google.common.base.Preconditions; @@ -117,6 +121,25 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { shufflePort); } + public Endpoint getServicesEndpoint() { + final int servicePort = + conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_PORT, + LlapConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT); + final boolean isSSL = + conf.getBoolean(LlapConfiguration.LLAP_DAEMON_SERVICE_SSL, + LlapConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT); + final String scheme = isSSL ? "https" : "http"; + final URL serviceURL; + try { + serviceURL = new URL(scheme, hostname, servicePort, ""); + return RegistryTypeUtils.webEndpoint("services", serviceURL.toURI()); + } catch (MalformedURLException e) { + throw new TezUncheckedException(e); + } catch (URISyntaxException e) { + throw new TezUncheckedException("llap service URI for " + hostname + " is invalid", e); + } + } + private final String getPath() { return this.path; } @@ -127,6 +150,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { ServiceRecord srv = new ServiceRecord(); srv.addInternalEndpoint(getRpcEndpoint()); srv.addInternalEndpoint(getShuffleEndpoint()); + srv.addExternalEndpoint(getServicesEndpoint()); for (Map.Entry<String, String> kv : this.conf) { if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX) http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java new file mode 100644 index 0000000..cb57c9a --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java @@ -0,0 +1,12 @@ +package org.apache.hadoop.hive.llap.daemon.services.impl; + +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.YarnWebParams; + +public class LlapWebApp extends WebApp { + + @Override + public void setup() { + // JMX / config are defaults + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java new file mode 100644 index 0000000..2275719 --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java @@ -0,0 +1,53 @@ +package org.apache.hadoop.hive.llap.daemon.services.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; +import org.apache.log4j.Logger; + +public class LlapWebServices extends AbstractService { + + private static final Logger LOG = Logger.getLogger(LlapWebServices.class); + + private int port; + private boolean ssl; + private Configuration conf; + private WebApp webApp; + private LlapWebApp webAppInstance; + + public LlapWebServices() { + super("LlapWebServices"); + } + + @Override + public void serviceInit(Configuration conf) { + + this.conf = new Configuration(conf); + this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + + this.port = conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_PORT, + LlapConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT); + this.ssl = conf.getBoolean(LlapConfiguration.LLAP_DAEMON_SERVICE_SSL, + LlapConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT); + + this.webAppInstance = new LlapWebApp(); + } + + @Override + public void serviceStart() throws Exception { + String bindAddress = "0.0.0.0"; + this.webApp = + WebApps.$for("llap").at(bindAddress).at(port).with(getConfig()) + /* TODO: security negotiation here */ + .start(); + } + + public void serviceStop() throws Exception { + if (this.webApp != null) { + this.webApp.stop(); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/main/resources/webapps/llap/.keep ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/webapps/llap/.keep b/llap-server/src/main/resources/webapps/llap/.keep new file mode 100644 index 0000000..e69de29
