Repository: incubator-eagle Updated Branches: refs/heads/master dc9fe0b47 -> a267e731e
[EAGLE-785] update hbase configuration to optional https://issues.apache.org/jira/browse/EAGLE-785 Author: Zhao, Qingwen <qingwz...@apache.org> Closes #674 from qingwen220/EAGLE-785. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a267e731 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a267e731 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a267e731 Branch: refs/heads/master Commit: a267e731e13b2b382a1a115a65259f96ee0b4e5f Parents: dc9fe0b Author: Zhao, Qingwen <qingwz...@apache.org> Authored: Tue Nov 22 15:14:22 2016 +0800 Committer: Zhao, Qingwen <qingwz...@apache.org> Committed: Tue Nov 22 15:14:22 2016 +0800 ---------------------------------------------------------------------- .../eagle/topology/TopologyCheckAppConfig.java | 9 ++-- .../hbase/HbaseTopologyEntityParser.java | 4 +- ....eagle.topology.TopologyCheckAppProvider.xml | 52 ++++++++------------ .../src/main/resources/application.conf | 9 ++-- 4 files changed, 33 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a267e731/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java index 0234a4d..a1c65a9 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java @@ -85,16 +85,17 @@ public class TopologyCheckAppConfig implements Serializable { //e.printStackTrace(); } - if (config.hasPath("dataSourceConfig.hbase")) { + if (config.getBoolean("dataSourceConfig.hbase.enabled")) { topologyTypes.add(TopologyConstants.TopologyType.HBASE); hBaseConfig = new HBaseConfig(); - hBaseConfig.eagleKeytab = config.getString("dataSourceConfig.hbase.kerberos.eagle.keytab"); - hBaseConfig.eaglePrincipal = config.getString("dataSourceConfig.hbase.kerberos.eagle.principal"); - hBaseConfig.hbaseMasterPrincipal = config.getString("dataSourceConfig.hbase.kerberos.master.principal"); + hBaseConfig.zkQuorum = config.getString("dataSourceConfig.hbase.zkQuorum"); hBaseConfig.zkRoot = config.getString("dataSourceConfig.hbase.zkZnodeParent"); hBaseConfig.zkClientPort = getOptionalConfig("dataSourceConfig.hbase.zkPropertyClientPort", HBASE_ZOOKEEPER_CLIENT_PORT); hBaseConfig.zkRetryTimes = getOptionalConfig("dataSourceConfig.hbase.zkRetryTimes", "5"); + hBaseConfig.eagleKeytab = getOptionalConfig("dataSourceConfig.hbase.kerberos.eagle.keytab", null); + hBaseConfig.eaglePrincipal = getOptionalConfig("dataSourceConfig.hbase.kerberos.eagle.principal", null); + hBaseConfig.hbaseMasterPrincipal = getOptionalConfig("dataSourceConfig.hbase.kerberos.master.principal", null); } if (config.hasPath("dataSourceConfig.mr")) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a267e731/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java index f93665d..77781a3 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java @@ -55,10 +55,10 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser { this.hBaseConfiguration.set("zookeeper.znode.parent", hBaseConfig.zkRoot); this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes); // kerberos authentication - this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal); - this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab); if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) { + this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal); + this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab); this.kerberosEnable = true; this.hBaseConfiguration.set("hbase.security.authentication", "kerberos"); this.hBaseConfiguration.set("hbase.master.kerberos.principal", hBaseConfig.hbaseMasterPrincipal); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a267e731/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml index 006971e..0b83e9b 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml +++ b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml @@ -26,25 +26,25 @@ <property> <name>dataExtractorConfig.fetchDataIntervalInSecs</name> <displayName>Fetch Data Interval in Secs</displayName> - <description>Fetch Data Interval in Secs</description> + <description>fetch data interval in secs</description> <value>300</value> </property> <property> <name>dataExtractorConfig.parseThreadPoolSize</name> <displayName>Parser Thread Pool Size</displayName> - <description>Parser Thread Pool Size</description> + <description>parser thread pool size</description> <value>5</value> </property> <property> <name>dataExtractorConfig.numDataFetcherSpout</name> <displayName>Spout Task Number</displayName> - <description>Spout Task Number</description> + <description>spout task number</description> <value>1</value> </property> <property> <name>dataExtractorConfig.numEntityPersistBolt</name> <displayName>Bolt Task Number</displayName> - <description>Bolt Task Number</description> + <description>bolt task number</description> <value>1</value> </property> @@ -58,57 +58,47 @@ <property> <name>dataSourceConfig.hbase.zkQuorum</name> <displayName>HBase Zookeeper Quorum</displayName> - <description>HBase Zookeeper Quorum</description> + <description>hbase zookeeper quorum (optional)</description> </property> <property> <name>dataSourceConfig.hbase.zkZnodeParent</name> <displayName>Hbase Zookeeper Znode Parent Root</displayName> - <description>Hbase Zookeeper Znode Parent Root</description> + <description>hbase zookeeper znode parent root (optional)</description> </property> <property> <name>dataSourceConfig.hbase.zkPropertyClientPort</name> <displayName>Hbase Zookeeper Client Port</displayName> - <description>Hbase Zookeeper Client Port</description> - </property> - <property> - <name>dataSourceConfig.hbase.kerberos.master.principal</name> - <displayName>Hbase Master Principal</displayName> - <description>Hbase Master Principal</description> + <description>hbase zookeeper client port (optional)</description> </property> - <property> - <name>dataSourceConfig.hbase.kerberos.eagle.principal</name> - <displayName>Hbase Eagle Principal</displayName> - <description>Hbase Master Principal</description> - <value></value> - </property> - <property> - <name>dataSourceConfig.hbase.kerberos.eagle.keytab</name> - <displayName>Hbase Eagle KeyTab</displayName> - <description>Hbase Eagle KeyTab</description> - <value></value> - </property> <property> <name>dataSourceConfig.hdfs.namenodeUrl</name> <displayName>Hdfs Namenode Web URL</displayName> - <description>Hdfs Namenode Web URL</description> + <description>hdfs namenode web url for HDFS monitor</description> <value>http://sandbox.hortonworks.com:50070</value> <required>true</required> </property> <property> <name>dataSourceConfig.mr.rmUrl</name> <displayName>Resource Manager URL</displayName> - <description>Resource Manager URL</description> + <description>resource manager url for YARN monitor</description> <value>http://sandbox.hortonworks.com:8088</value> <required>true</required> </property> <property> <name>dataSourceConfig.mr.historyServerUrl</name> <displayName>History Server URL</displayName> - <description>History Server URL</description> + <description>URL for history server monitor (optional)</description> <value></value> </property> + <property> + <name>dataSourceConfig.hbase.enabled</name> + <displayName>HBase Config Enabled</displayName> + <description>enabled for HBase monitor</description> + <value>false</value> + <required>true</required> + </property> <property> <name>topology.numOfSinkTasks</name> @@ -122,25 +112,25 @@ <name>dataSinkConfig.topic</name> <displayName>Topic For Kafka Data Sink</displayName> <value>topology_health_check</value> - <description>Topic For Kafka Data Sink</description> + <description>topic For kafka data sink</description> </property> <property> <name>dataSinkConfig.brokerList</name> <displayName>Kafka Broker List</displayName> <value>server.eagle.apache.org:9092</value> - <description>Kafka Broker List</description> + <description>kafka broker list</description> </property> <property> <name>dataSinkConfig.serializerClass</name> <displayName>Serializer Class Kafka Message Value</displayName> <value>kafka.serializer.StringEncoder</value> - <description>Serializer Class Kafka Message Value</description> + <description>serializer class kafka message value</description> </property> <property> <name>dataSinkConfig.keySerializerClass</name> <displayName>Serializer Class Kafka Message Key</displayName> <value>kafka.serializer.StringEncoder</value> - <description>Serializer Class Kafka Message Key</description> + <description>serializer class kafka message key</description> </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a267e731/eagle-topology-check/eagle-topology-app/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/resources/application.conf b/eagle-topology-check/eagle-topology-app/src/main/resources/application.conf index 1795849..4d22912 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/resources/application.conf +++ b/eagle-topology-check/eagle-topology-app/src/main/resources/application.conf @@ -34,7 +34,12 @@ dataSourceConfig : { hdfs.namenodeUrl: "http://sandbox.hortonworks.com:50070";, + mr: { + rmUrl: "http://sandbox.hortonworks.com:50030";, + historyServerUrl : "http://sandbox.hortonworks.com:19888"; #if not need, then empty + } hbase: { + enabled: false, zkQuorum: "sandbox.hortonworks.com", zkPropertyClientPort : "2181", zkZnodeParent: "/hbase-unsecure", @@ -45,10 +50,6 @@ eagle.keytab: "" } }, - mr: { - rmUrl: "http://sandbox.hortonworks.com:50030";, - historyServerUrl : "http://sandbox.hortonworks.com:19888"; #if not need, then empty - } } "dataSinkConfig": {
