This is an automated email from the ASF dual-hosted git repository. ajay pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 7f636b4 HDDS-919. Enable prometheus endpoints for Ozone datanodes (#502) 7f636b4 is described below commit 7f636b47b5e430e340c8fd731fd9e35e22f9ddbf Author: Elek, Márton <e...@users.noreply.github.com> AuthorDate: Tue Mar 5 21:04:57 2019 +0100 HDDS-919. Enable prometheus endpoints for Ozone datanodes (#502) Pull request from Elek, Márton. (#502) --- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 20 +++++ .../common/src/main/resources/ozone-default.xml | 66 ++++++++++++++++- .../hadoop/ozone/HddsDatanodeHttpServer.java | 86 ++++++++++++++++++++++ .../apache/hadoop/ozone/HddsDatanodeService.java | 16 ++++ .../main/resources/webapps/hddsDatanode/.gitkeep | 17 +++++ .../src/main/compose/ozone/docker-compose.yaml | 1 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 + 7 files changed, 206 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index c265771..184e3d8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -227,4 +227,24 @@ public final class HddsConfigKeys { public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL = "hdds.security.client.scm.certificate.protocol.acl"; + public static final String HDDS_DATANODE_HTTP_ENABLED_KEY = + "hdds.datanode.http.enabled"; + public static final String HDDS_DATANODE_HTTP_BIND_HOST_KEY = + "hdds.datanode.http-bind-host"; + public static final String HDDS_DATANODE_HTTPS_BIND_HOST_KEY = + "hdds.datanode.https-bind-host"; + public static final String HDDS_DATANODE_HTTP_ADDRESS_KEY = + "hdds.datanode.http-address"; + public static final String HDDS_DATANODE_HTTPS_ADDRESS_KEY = + "hdds.datanode.https-address"; + + public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; + public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882; + public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883; + public static final String + HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY = + "hdds.datanode.http.kerberos.principal"; + public static final String + HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY = + "hdds.datanode.http.kerberos.keytab"; } \ No newline at end of file diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f7fecb7..a4f49e7 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2030,6 +2030,70 @@ </property> <property> + <name>hdds.datanode.http.kerberos.principal</name> + <value>HTTP/_h...@example.com</value> + <tag>HDDS, SECURITY, MANAGEMENT</tag> + <description> + The kerberos principal for the datanode http server. + </description> + </property> + <property> + <name>hdds.datanode.http.kerberos.keytab</name> + <value>/etc/security/keytabs/HTTP.keytab</value> + <tag>HDDS, SECURITY, MANAGEMENT</tag> + <description> + The kerberos keytab file for datanode http server + </description> + </property> + <property> + <name>hdds.datanode.http-address</name> + <value>0.0.0.0:9882</value> + <tag>HDDS, MANAGEMENT</tag> + <description> + The address and the base port where the Datanode web ui will listen on. + If the port is 0 then the server will start on a free port. + </description> + </property> + <property> + <name>hdds.datanode.http-bind-host</name> + <value>0.0.0.0</value> + <tag>HDDS, MANAGEMENT</tag> + <description> + The actual address the Datanode web server will bind to. If this + optional address is set, it overrides only the hostname portion of + hdds.datanode.http-address. + </description> + </property> + <property> + <name>hdds.datanode.http.enabled</name> + <value>true</value> + <tag>HDDS, MANAGEMENT</tag> + <description> + Property to enable or disable Datanode web ui. + </description> + </property> + <property> + <name>hdds.datanode.https-address</name> + <value>0.0.0.0:9883</value> + <tag>HDDS, MANAGEMENT, SECURITY</tag> + <description> + The address and the base port where the Datanode web UI will listen + on using HTTPS. + + If the port is 0 then the server will start on a free port. + </description> + </property> + <property> + <name>hdds.datanode.https-bind-host</name> + <value>0.0.0.0</value> + <tag>HDDS, MANAGEMENT, SECURITY</tag> + <description> + The actual address the Datanode web server will bind to using HTTPS. + If this optional address is set, it overrides only the hostname portion of + hdds.datanode.http-address. + </description> + </property> + <property> <name>ozone.client.retry.max.attempts</name> <value>10</value> <description> @@ -2070,4 +2134,4 @@ milliseconds. </description> </property> -</configuration> \ No newline at end of file +</configuration> diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java new file mode 100644 index 0000000..3dcfcfe --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.server.BaseHttpServer; + +/** + * Simple http server to provide basic monitoring for hdds datanode. + * <p> + * This server is used to access default /conf /prom /prof endpoints. + */ +public class HddsDatanodeHttpServer extends BaseHttpServer { + + public HddsDatanodeHttpServer(Configuration conf) throws IOException { + super(conf, "hddsDatanode"); + } + + @Override + protected String getHttpAddressKey() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY; + } + + @Override + protected String getHttpBindHostKey() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_KEY; + } + + @Override + protected String getHttpsAddressKey() { + return HddsConfigKeys.HDDS_DATANODE_HTTPS_ADDRESS_KEY; + } + + @Override + protected String getHttpsBindHostKey() { + return HddsConfigKeys.HDDS_DATANODE_HTTPS_BIND_HOST_KEY; + } + + @Override + protected String getBindHostDefault() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT; + } + + @Override + protected int getHttpBindPortDefault() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT; + } + + @Override + protected int getHttpsBindPortDefault() { + return HddsConfigKeys.HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT; + } + + @Override + protected String getKeytabFile() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY; + } + + @Override + protected String getSpnegoPrincipal() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY; + } + + @Override + protected String getEnabledKey() { + return HddsConfigKeys.HDDS_DATANODE_HTTP_ENABLED_KEY; + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 3a92a4a..3c205e6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; + import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsUtils; @@ -67,6 +68,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin { private DatanodeDetails datanodeDetails; private DatanodeStateMachine datanodeStateMachine; private List<ServicePlugin> plugins; + private HddsDatanodeHttpServer httpServer; /** * Default constructor. @@ -180,6 +182,12 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin { LOG.info("Hdds Datanode login successful."); } datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf); + try { + httpServer = new HddsDatanodeHttpServer(conf); + httpServer.start(); + } catch (Exception ex) { + LOG.error("HttpServer failed to start.", ex); + } startPlugins(); // Starting HDDS Daemons datanodeStateMachine.startDaemon(); @@ -294,6 +302,14 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin { if (datanodeStateMachine != null) { datanodeStateMachine.stopDaemon(); } + if (httpServer != null) { + try { + httpServer.stop(); + } catch (Exception e) { + LOG.error("Stopping HttpServer is failed.", e); + } + } + } @Override diff --git a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep new file mode 100644 index 0000000..ff1232e --- /dev/null +++ b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep @@ -0,0 +1,17 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index 487c4eb..9110592 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -23,6 +23,7 @@ services: - ../..:/opt/hadoop ports: - 9864 + - 9882 command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - ./docker-config diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 0ed8e8b..dbeb0b5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -577,6 +577,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster { private void configureHddsDatanodes() { conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY, "0.0.0.0:0"); + conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.set(HDDS_DATANODE_PLUGINS_KEY, "org.apache.hadoop.ozone.web.OzoneHddsDatanodeService"); conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org