Repository: incubator-eagle Updated Branches: refs/heads/master 8042f9e44 -> de061d599
[EAGLE-799] Can not get hbase info by service api when the master becomes not available Can not get hbase info by service api when the master becomes not available Author: yupu <y...@ebay.com> Closes #680 from puyulu/EAGLE-799. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/de061d59 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/de061d59 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/de061d59 Branch: refs/heads/master Commit: de061d599c35735abd6b9447d37949289827e1e3 Parents: 8042f9e Author: yupu <y...@ebay.com> Authored: Thu Nov 24 21:14:43 2016 +0800 Committer: Zhao, Qingwen <qingwz...@apache.org> Committed: Thu Nov 24 21:14:43 2016 +0800 ---------------------------------------------------------------------- .../extractor/hbase/HbaseTopologyEntityParser.java | 5 +++-- .../extractor/hdfs/HdfsTopologyEntityParser.java | 13 +++++++++++++ .../topology/extractor/mr/MRTopologyEntityParser.java | 9 +++++---- .../eagle/topology/storm/TopologyDataPersistBolt.java | 2 +- .../topology/entity/HdfsServiceTopologyAPIEntity.java | 13 +++++++++++++ 5 files changed, 35 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java index 77781a3..f3eac23 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java @@ -46,7 +46,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser { private Boolean kerberosEnable = false; private TopologyRackResolver rackResolver; - public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) { + public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) { this.site = site; this.rackResolver = resolver; this.hBaseConfiguration = HBaseConfiguration.create(); @@ -56,7 +56,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser { this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes); // kerberos authentication if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null - && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) { + && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) { this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal); this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab); this.kerberosEnable = true; @@ -103,6 +103,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser { } double liveRatio = liveServers * 1d / (liveServers + deadServers); result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE, liveRatio, site, timestamp)); + result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE, 1d, site, timestamp)); return result; } catch (RuntimeException e) { e.printStackTrace(); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java index 627584a..72dce80 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java @@ -72,6 +72,8 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser { private static final String DATA_NODE_CAPACITY = "capacity"; private static final String DATA_NODE_ADMIN_STATE = "adminState"; private static final String DATA_NODE_FAILED_VOLUMN = "volfails"; + private static final String DATA_NODE_VERSION = "version"; + private static final String NAME_NODE_VERSION = "Version"; private static final String DATA_NODE_DECOMMISSIONED = "Decommissioned"; private static final String DATA_NODE_DECOMMISSIONED_STATE = "decommissioned"; @@ -115,9 +117,18 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser { final String urlString = buildFSNamesystemURL(url); final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString); final JMXBean bean = jmxBeanMap.get(JMX_FS_NAME_SYSTEM_BEAN_NAME); + + final String nameNodeUrlString = buildNamenodeInfo(url); + final Map<String, JMXBean> nameNodeMap = JMXQueryHelper.query(nameNodeUrlString); + final JMXBean nameNodeBean = nameNodeMap.get(JMX_NAMENODE_INFO); + if (bean == null || bean.getPropertyMap() == null) { throw new ServiceNotResponseException("Invalid JMX format, FSNamesystem bean is null!"); } + if (nameNodeBean == null || nameNodeBean.getPropertyMap() == null) { + throw new ServiceNotResponseException("Invalid JMX format, NameNode bean is null!"); + } + final String hostname = (String) bean.getPropertyMap().get(HA_NAME); HdfsServiceTopologyAPIEntity result = createHdfsServiceEntity(TopologyConstants.NAME_NODE_ROLE, hostname, updateTime); final String state = (String) bean.getPropertyMap().get(HA_STATE); @@ -128,6 +139,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser { result.setUsedCapacityTB(Double.toString(capacityUsedGB / 1024)); final Integer blocksTotal = (Integer) bean.getPropertyMap().get(BLOCKS_TOTAL); result.setNumBlocks(Integer.toString(blocksTotal)); + result.setVersion((String) nameNodeBean.getPropertyMap().get(NAME_NODE_VERSION)); return result; } @@ -240,6 +252,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser { } else { entity.setStatus(TopologyConstants.DATA_NODE_LIVE_STATUS); } + entity.setVersion(String.valueOf(liveNode.get(DATA_NODE_VERSION))); numLiveNodes++; result.getSlaveNodes().add(entity); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java index 671d07b..52148b7 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java @@ -46,7 +46,7 @@ import static org.apache.eagle.topology.TopologyConstants.*; public class MRTopologyEntityParser implements TopologyEntityParser { - private String [] rmUrls; + private String[] rmUrls; private String historyServerUrl; private String site; private TopologyRackResolver rackResolver; @@ -91,9 +91,10 @@ public class MRTopologyEntityParser implements TopologyEntityParser { // reSelect url } } - if (result.getMasterNodes().isEmpty()) { - result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.RESOURCE_MANAGER_ROLE, 0, site, timestamp)); - } + + double value = result.getMasterNodes().isEmpty() ? 0 : result.getMasterNodes().size() * 1d / rmUrls.length; + result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.RESOURCE_MANAGER_ROLE, value, site, timestamp)); + doCheckHistoryServer(timestamp, result); return result; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java index e0ff3cd..afd3518 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java @@ -90,7 +90,7 @@ public class TopologyDataPersistBolt extends BaseRichBolt { GenericServiceAPIResponseEntity<TopologyBaseAPIEntity> response = client.search().query(query).pageSize(Integer.MAX_VALUE).send(); if (response.isSuccess() && response.getObj() != null) { for (TopologyBaseAPIEntity entity : response.getObj()) { - if (!availableHostnames.contains(generateKey(entity))) { + if (availableHostnames != null && availableHostnames.size() > 0 && !availableHostnames.contains(generateKey(entity))) { entitiesForDeletion.add(entity); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java index baca97b..d5ee41e 100644 --- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java +++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java @@ -43,6 +43,8 @@ public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity { private long writtenTxidDiff; @Column("g") private long lastUpdateTime; + @Column("h") + private String version; public long getLastUpdateTime() { return lastUpdateTime; @@ -106,5 +108,16 @@ public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity { this.writtenTxidDiff = writtenTxidDiff; valueChanged("writtenTxidDiff"); } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + valueChanged("version"); + } + + }
