AMBARI-21939 : Ambari HDFS Heatmaps are not showing data (Not Applicable) for bytes read, bytes written & DataNode Process Network I/O Utilization. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a49121b9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a49121b9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a49121b9 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: a49121b9af276f45ef26e0b320de48d6ef6e7563 Parents: 7fa7a6c Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Sep 12 17:07:14 2017 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Tue Sep 12 17:07:14 2017 -0700 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog260.java | 16 ++++ .../PERF/1.0/services/FAKEHDFS/widgets.json | 48 +++++------ .../server/upgrade/UpgradeCatalog260Test.java | 87 ++++++++++++++++++++ 3 files changed, 127 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a49121b9/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java index 2de85fc..b17569e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java @@ -21,6 +21,7 @@ import static org.apache.ambari.server.view.ViewContextImpl.CORE_SITE; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -401,6 +402,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { ensureZeppelinProxyUserConfigs(); updateKerberosDescriptorArtifacts(); updateAmsConfigs(); + updateHDFSWidgetDefinition(); } public int getCurrentVersionID() throws AmbariException, SQLException { @@ -565,4 +567,18 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { } } } + + protected void updateHDFSWidgetDefinition() throws AmbariException { + LOG.info("Updating HDFS widget definition."); + + Map<String, List<String>> widgetMap = new HashMap<>(); + Map<String, String> sectionLayoutMap = new HashMap<>(); + + List<String> hdfsHeatmapWidgets = new ArrayList<>(Arrays.asList("HDFS Bytes Read", "HDFS Bytes Written", + "DataNode Process Disk I/O Utilization", "DataNode Process Network I/O Utilization")); + widgetMap.put("HDFS_HEATMAPS", hdfsHeatmapWidgets); + sectionLayoutMap.put("HDFS_HEATMAPS", "default_hdfs_heatmap"); + + updateWidgetDefinitionsForService("HDFS", widgetMap, sectionLayoutMap); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/a49121b9/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/widgets.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/widgets.json b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/widgets.json index 7a793f8..205a364 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/widgets.json +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/widgets.json @@ -416,8 +416,8 @@ "is_visible": true, "metrics": [ { - "name": "dfs.datanode.BytesRead._rate", - "metric_path": "metrics/dfs/datanode/bytes_read._rate", + "name": "dfs.datanode.BytesRead", + "metric_path": "metrics/dfs/datanode/bytes_read", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" } @@ -425,7 +425,7 @@ "values": [ { "name": "FAKEHDFS Bytes Read", - "value": "${dfs.datanode.BytesRead._rate}" + "value": "${dfs.datanode.BytesRead}" } ], "properties": { @@ -440,8 +440,8 @@ "is_visible": false, "metrics": [ { - "name": "dfs.datanode.BytesWritten._rate", - "metric_path": "metrics/dfs/datanode/bytes_written._rate", + "name": "dfs.datanode.BytesWritten", + "metric_path": "metrics/dfs/datanode/bytes_written", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" } @@ -449,7 +449,7 @@ "values": [ { "name": "FAKEHDFS Bytes Written", - "value": "${dfs.datanode.BytesWritten._rate}" + "value": "${dfs.datanode.BytesWritten}" } ], "properties": { @@ -537,26 +537,26 @@ "is_visible": false, "metrics": [ { - "name": "dfs.datanode.BytesRead._rate", - "metric_path": "metrics/dfs/datanode/bytes_read._rate", + "name": "dfs.datanode.BytesRead", + "metric_path": "metrics/dfs/datanode/bytes_read", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" }, { - "name": "dfs.datanode.BytesWritten._rate", - "metric_path": "metrics/dfs/datanode/bytes_written._rate", + "name": "dfs.datanode.BytesWritten", + "metric_path": "metrics/dfs/datanode/bytes_written", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" }, { - "name": "dfs.datanode.TotalReadTime._rate", - "metric_path": "metrics/dfs/datanode/TotalReadTime._rate", + "name": "dfs.datanode.TotalReadTime", + "metric_path": "metrics/dfs/datanode/TotalReadTime", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" }, { - "name": "dfs.datanode.TotalWriteTime._rate", - "metric_path": "metrics/dfs/datanode/TotalWriteTime._rate", + "name": "dfs.datanode.TotalWriteTime", + "metric_path": "metrics/dfs/datanode/TotalWriteTime", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" } @@ -564,7 +564,7 @@ "values": [ { "name": "FAKEDataNode Process Disk I/O Utilization", - "value": "${((dfs.datanode.BytesRead._rate/dfs.datanode.TotalReadTime._rate)+(dfs.datanode.BytesWritten._rate/dfs.datanode.TotalWriteTime._rate))*50}" + "value": "${((dfs.datanode.BytesRead/dfs.datanode.TotalReadTime)+(dfs.datanode.BytesWritten/dfs.datanode.TotalWriteTime))*50}" } ], "properties": { @@ -579,26 +579,26 @@ "is_visible": false, "metrics": [ { - "name": "dfs.datanode.RemoteBytesRead._rate", - "metric_path": "metrics/dfs/datanode/RemoteBytesRead._rate", + "name": "dfs.datanode.RemoteBytesRead", + "metric_path": "metrics/dfs/datanode/RemoteBytesRead", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" }, { - "name": "dfs.datanode.ReadsFromRemoteClient._rate", - "metric_path": "metrics/dfs/datanode/reads_from_remote_client._rate", + "name": "dfs.datanode.ReadsFromRemoteClient", + "metric_path": "metrics/dfs/datanode/reads_from_remote_client", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" }, { - "name": "dfs.datanode.RemoteBytesWritten._rate", - "metric_path": "metrics/dfs/datanode/RemoteBytesWritten._rate", + "name": "dfs.datanode.RemoteBytesWritten", + "metric_path": "metrics/dfs/datanode/RemoteBytesWritten", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" }, { - "name": "dfs.datanode.WritesFromRemoteClient._rate", - "metric_path": "metrics/dfs/datanode/writes_from_remote_client._rate", + "name": "dfs.datanode.WritesFromRemoteClient", + "metric_path": "metrics/dfs/datanode/writes_from_remote_client", "service_name": "FAKEHDFS", "component_name": "FAKEDATANODE" } @@ -606,7 +606,7 @@ "values": [ { "name": "FAKEDataNode Process Network I/O Utilization", - "value": "${((dfs.datanode.RemoteBytesRead._rate/dfs.datanode.ReadsFromRemoteClient._rate)+(dfs.datanode.RemoteBytesWritten._rate/dfs.datanode.WritesFromRemoteClient._rate))*50}" + "value": "${((dfs.datanode.RemoteBytesRead/dfs.datanode.ReadsFromRemoteClient)+(dfs.datanode.RemoteBytesWritten/dfs.datanode.WritesFromRemoteClient))*50}" } ], "properties": { http://git-wip-us.apache.org/repos/asf/ambari/blob/a49121b9/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java index b70f37b..d9c7957 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java @@ -50,8 +50,10 @@ import java.util.Set; import javax.persistence.EntityManager; import com.google.common.collect.Maps; +import com.google.inject.AbstractModule; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariManagementControllerImpl; @@ -60,17 +62,23 @@ import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; import org.apache.ambari.server.orm.dao.ArtifactDAO; +import org.apache.ambari.server.orm.dao.WidgetDAO; import org.apache.ambari.server.orm.entities.ArtifactEntity; +import org.apache.ambari.server.orm.entities.WidgetEntity; +import org.apache.ambari.server.stack.StackManagerFactory; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.commons.io.FileUtils; import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.EasyMockRunner; @@ -80,7 +88,9 @@ import org.easymock.MockType; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import com.google.gson.Gson; @@ -142,6 +152,9 @@ public class UpgradeCatalog260Test { @Mock(type = MockType.NICE) private Injector injector; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void init() { reset(entityManagerProvider, injector); @@ -737,4 +750,78 @@ public class UpgradeCatalog260Test { assertTrue(Maps.difference(newProperties, updatedProperties).areEqual()); } + @Test + public void testHDFSWidgetUpdate() throws Exception { + final Clusters clusters = createNiceMock(Clusters.class); + final Cluster cluster = createNiceMock(Cluster.class); + final AmbariManagementController controller = createNiceMock(AmbariManagementController.class); + final Gson gson = new Gson(); + final WidgetDAO widgetDAO = createNiceMock(WidgetDAO.class); + final AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class); + WidgetEntity widgetEntity = createNiceMock(WidgetEntity.class); + StackId stackId = new StackId("HDP", "2.0.0"); + StackInfo stackInfo = createNiceMock(StackInfo.class); + ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class); + Service service = createNiceMock(Service.class); + + String widgetStr = "{\n" + + " \"layouts\": [\n" + + " {\n" + + " \"layout_name\": \"default_hdfs_heatmap\",\n" + + " \"display_name\": \"Standard HDFS HeatMaps\",\n" + + " \"section_name\": \"HDFS_HEATMAPS\",\n" + + " \"widgetLayoutInfo\": [\n" + + " {\n" + + " \"widget_name\": \"HDFS Bytes Read\",\n" + + " \"metrics\": [],\n" + + " \"values\": []\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + + File dataDirectory = temporaryFolder.newFolder(); + File file = new File(dataDirectory, "hdfs_widget.json"); + FileUtils.writeStringToFile(file, widgetStr); + + final Injector mockInjector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class)); + bind(AmbariManagementController.class).toInstance(controller); + bind(Clusters.class).toInstance(clusters); + bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + bind(Gson.class).toInstance(gson); + bind(WidgetDAO.class).toInstance(widgetDAO); + bind(StackManagerFactory.class).toInstance(createNiceMock(StackManagerFactory.class)); + bind(AmbariMetaInfo.class).toInstance(metaInfo); + } + }); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).anyTimes(); + expect(cluster.getServices()).andReturn(Collections.singletonMap("HDFS", service)).anyTimes(); + expect(cluster.getClusterId()).andReturn(1L).anyTimes(); + expect(service.getDesiredStackId()).andReturn(stackId).anyTimes(); + expect(stackInfo.getService("HDFS")).andReturn(serviceInfo); + expect(cluster.getDesiredStackVersion()).andReturn(stackId).anyTimes(); + expect(metaInfo.getStack("HDP", "2.0.0")).andReturn(stackInfo).anyTimes(); + expect(serviceInfo.getWidgetsDescriptorFile()).andReturn(file).anyTimes(); + + expect(widgetDAO.findByName(1L, "HDFS Bytes Read", "ambari", "HDFS_HEATMAPS")) + .andReturn(Collections.singletonList(widgetEntity)); + expect(widgetDAO.merge(widgetEntity)).andReturn(null); + expect(widgetEntity.getWidgetName()).andReturn("HDFS Bytes Read").anyTimes(); + + replay(clusters, cluster, controller, widgetDAO, metaInfo, widgetEntity, stackInfo, serviceInfo, service); + + mockInjector.getInstance(UpgradeCatalog260.class).updateHDFSWidgetDefinition(); + + verify(clusters, cluster, controller, widgetDAO, widgetEntity, stackInfo, serviceInfo); + } + + }