[EAGLE-952] system dashbord https://issues.apache.org/jira/browse/EAGLE-952
Author: koone <[email protected]> Closes #871 from koone/EAGLE-952. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/877d3e77 Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/877d3e77 Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/877d3e77 Branch: refs/heads/master Commit: 877d3e77b18c162455d1970a0cff9d5e04dc44b2 Parents: fa713a4 Author: koone <[email protected]> Authored: Thu Mar 16 08:10:51 2017 +0000 Committer: r7raul1984 <[email protected]> Committed: Thu Mar 16 08:10:51 2017 +0000 ---------------------------------------------------------------------- eagle-metric/eagle-system-web/pom.xml | 61 ++++ .../SystemMetricWebApplicationProvider.java | 22 ++ ...etric.SystemMetricWebApplicationProvider.xml | 46 +++ ...org.apache.eagle.app.spi.ApplicationProvider | 16 + .../src/main/webapp/app/apps/system/config.json | 80 +++++ .../webapp/app/apps/system/ctrl/overview.js | 321 +++++++++++++++++++ .../app/apps/system/ctrl/serverDetailCtrl.js | 229 +++++++++++++ .../app/apps/system/ctrl/serverListCtrl.js | 39 +++ .../src/main/webapp/app/apps/system/index.html | 6 + .../src/main/webapp/app/apps/system/index.js | 293 +++++++++++++++++ .../app/apps/system/partials/overview.html | 126 ++++++++ .../app/apps/system/partials/serverDetail.html | 111 +++++++ .../app/apps/system/partials/serverList.html | 60 ++++ .../main/webapp/app/apps/system/style/index.css | 88 +++++ .../app/apps/system/widget/availabilityChart.js | 138 ++++++++ .../src/main/webapp/package.json | 0 eagle-metric/pom.xml | 3 +- eagle-server/pom.xml | 10 +- ...org.apache.eagle.app.spi.ApplicationProvider | 3 +- eagle-topology-check/eagle-topology-app/pom.xml | 5 + .../apache/eagle/topology/TopologyCheckApp.java | 27 +- .../eagle/topology/TopologyCheckAppConfig.java | 20 ++ .../system/SystemCheckPersistBolt.java | 139 ++++++++ ....eagle.topology.TopologyCheckAppProvider.xml | 32 ++ .../eagle/topology/TopologyConstants.java | 2 + .../entity/SystemServiceTopologyAPIEntity.java | 122 +++++++ .../entity/TopologyEntityRepository.java | 2 + 27 files changed, 1994 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/pom.xml b/eagle-metric/eagle-system-web/pom.xml new file mode 100644 index 0000000..3a762af --- /dev/null +++ b/eagle-metric/eagle-system-web/pom.xml @@ -0,0 +1,61 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>eagle-metric-parent</artifactId> + <groupId>org.apache.eagle</groupId> + <version>0.5.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>eagle-system-web</artifactId> + <name>Eagle::App::HadoopMetric::System::WebUI</name> + <dependencies> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-app-base</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-hadoop-metric</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-topology-app</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <resources> + <resource> + <directory>src/main/webapp/app</directory> + <targetPath>assets/</targetPath> + </resource> + <resource> + <directory>src/main/resources</directory> + </resource> + </resources> + </build> + +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/java/org/apache/eagle/metric/SystemMetricWebApplicationProvider.java ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/java/org/apache/eagle/metric/SystemMetricWebApplicationProvider.java b/eagle-metric/eagle-system-web/src/main/java/org/apache/eagle/metric/SystemMetricWebApplicationProvider.java new file mode 100644 index 0000000..ff0cb25 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/java/org/apache/eagle/metric/SystemMetricWebApplicationProvider.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.metric; + +import org.apache.eagle.app.StaticApplicationProvider; + +public class SystemMetricWebApplicationProvider extends StaticApplicationProvider { +} http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/resources/META-INF/providers/org.apache.eagle.metric.SystemMetricWebApplicationProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/resources/META-INF/providers/org.apache.eagle.metric.SystemMetricWebApplicationProvider.xml b/eagle-metric/eagle-system-web/src/main/resources/META-INF/providers/org.apache.eagle.metric.SystemMetricWebApplicationProvider.xml new file mode 100644 index 0000000..e53c182 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/resources/META-INF/providers/org.apache.eagle.metric.SystemMetricWebApplicationProvider.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<application> + <type>SYSTEM_METRIC_WEB_APP</type> + <name>System Metric Monitoring Web </name> + <viewPath>/apps/system</viewPath> + <description>System Metric Monitoring Web</description> + <dependencies> + <dependency> + <type>HADOOP_METRIC_MONITOR</type> + <required>true</required> + </dependency> + <dependency> + <type>TOPOLOGY_HEALTH_CHECK_APP</type> + <required>true</required> + </dependency> + </dependencies> + <configuration> + <property> + <name>service.host</name> + <displayName>Eagle Service Host</displayName> + <description>Set additional eagle service host, default: using current host</description> + </property> + <property> + <name>service.port</name> + <displayName>Eagle Service Port</displayName> + <description>Set additional eagle service port, default: using current port</description> + </property> + </configuration> +</application> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-metric/eagle-system-web/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider new file mode 100644 index 0000000..c7df1e8 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.eagle.metric.SystemMetricWebApplicationProvider \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/config.json ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/config.json b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/config.json new file mode 100644 index 0000000..7258147 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/config.json @@ -0,0 +1,80 @@ +{ + "totaleverage": { + + "memtotal" : "system.memory.memtotal.kb", + "memfree" : "system.memory.memfree.kb", + + "cpuidle" : "system.cpu.idle", + "cputotalusage" : "system.cpu.totalusage", + "1minloadavg" : "system.cpu.loadavg.1min", + "5minloadavg" : "system.cpu.loadavg.5min", + "15minloadavg" : "system.cpu.loadavg.15min", + + "receivedbytes" : "system.nic.receivedbytes", + "transmitbytes" : "system.nic.transmitbytes", + + "diskreadrate" : "system.disk.readrate", + "diskwriterate" : "system.disk.writerate", + "disktotal" : "system.disk.disktotal", + "diskused" : "system.disk.diskused" + }, + "systemnode": { + "cpuuser" : "system.cpu.user", + "cpunice" : "system.cpu.nice", + "cpusytem" : "system.cpu.system", + "cpuidle" : "system.cpu.idle", + "cpuwait" : "system.cpu.wait", + "cpuirq" : "system.cpu.irq", + "cpusoftirq" : "system.cpu.softirq", + "cpusteal" : "system.cpu.steal", + "cpuguest" : "system.cpu.guest", + "cpuusage" : "system.cpu.usage", + "cputotalusage" : "system.cpu.totalusage", + + "1minloadavg" : "system.cpu.loadavg.1min", + "5minloadavg" : "system.cpu.loadavg.5min", + "15minloadavg" : "system.cpu.loadavg.15min", + + "uptime" : "system.uptime.uptime.day", + "idletime" : "system.uptime.idletime.day", + + "memtotal" : "system.memory.memtotal.kb", + "memfree" : "system.memory.memfree.kb", + "memorybuffers" : "system.memory.buffers.kb", + "memorycached" : "system.memory.cached.kb", + "memoryswapcached" : "system.memory.swapcached.kb", + "memoryactive" : "system.memory.active.kb", + "memoryinactive" : "system.memory.inactive.kb", + "memoryactiveanon" : "system.memory.activeanon.kb", + "memoryinactiveanon" : "system.memory.inactiveanon.kb", + "memoryactivefile" : "system.memory.activefile.kb", + "memoryinactivefile" : "system.memory.inactivefile.kb", + "memoryunevictable" : "system.memory.unevictable.kb", + "memorymlocked" : "system.memory.mlocked.kb", + "memoryswaptotal" : "system.memory.swaptotal.kb", + "memoryswapfree" : "system.memory.swapfree.kb", + "memorydirty" : "system.memory.dirty.kb", + "memorypagetables" : "system.memory.pagetables.kb", + "memoryvmalloctotal" : "system.memory.vmalloctotal.kb", + "memoryvmallocused" : "system.memory.vmallocused.kb", + "memoryvmallocchunk" : "system.memory.vmallocchunk.kb", + "momoryusage" : "system.memory.usage", + + "receivedbytes" : "system.nic.receivedbytes", + "receivedpackets" : "system.nic.receivedpackets", + "receivederrs" : "system.nic.receivederrs", + "receiveddrop" : "system.nic.receiveddrop", + "transmitbytes" : "system.nic.transmitbytes", + "transmitpackets" : "system.nic.transmitpackets", + "transmiterrs" : "system.nic.transmiterrs", + "transmitdrop" : "system.nic.transmitdrop", + + "diskreadrate" : "system.disk.readrate", + "diskwriterate" : "system.disk.writerate", + "diskavgwaittime" : "system.disk.avgwaittime", + "diskutilization" : "system.disk.utilization", + "diskdisktotal" : "system.disk.disktotal", + "diskdiskused" : "system.disk.diskused", + "diskusage" : "system.disk.usage" + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/overview.js ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/overview.js b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/overview.js new file mode 100644 index 0000000..7a3b5e0 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/overview.js @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +(function () { + /** + * `register` without params will load the module which using require + */ + register(function (systemMetricApp) { + systemMetricApp.controller("overviewCtrl", function ($q, $wrapState, $scope, PageConfig, SYSTEMMETRIC, Time) { + var cache = {}; + $scope.site = $wrapState.param.siteId; + PageConfig.title = 'System Dashbord'; + + $scope.coresList = aggGroup($scope.site, ["cores"], "count"); + + var digitalKbOption = { + animation: false, + tooltip: { + formatter: function (points) { + return points[0].name + "<br/>" + + $.map(points, function (point) { + return '<span style="display:inline-block;margin-right:5px;border-radius:10px;width:9px;height:9px;background-color:' + point.color + '"></span> ' + + point.seriesName + ": " + + common.number.abbr(point.value / 1024 /1024, false, 0) + "Gb"; + }).reverse().join("<br/>"); + } + }, + yAxis: [{ + axisLabel: { + formatter: function (value) { + return common.number.abbr(value / 1024 /1024, false, 0); + } + } + }] + }; + + var digitalOption = { + animation: false, + tooltip: { + formatter: function (points) { + return points[0].name + "<br/>" + + $.map(points, function (point) { + return '<span style="display:inline-block;margin-right:5px;border-radius:10px;width:9px;height:9px;background-color:' + point.color + '"></span> ' + + point.seriesName + ": " + + common.number.abbr(point.value, false, 0); + }).reverse().join("<br/>"); + } + }, + yAxis: [{ + axisLabel: { + formatter: function (value) { + return common.number.abbr(value, false); + } + } + }] + }; + + var sizeoption = { + animation: false, + tooltip: { + formatter: function (points) { + return points[0].name + "<br/>" + + $.map(points, function (point) { + return '<span style="display:inline-block;margin-right:5px;border-radius:10px;width:9px;height:9px;background-color:' + point.color + '"></span> ' + + point.seriesName + ": " + + common.number.abbr(point.value, true); + }).reverse().join("<br/>"); + } + }, + legend: { + x: 'center', y: 'bottom' + }, + areaStyle: {normal: {}}, + yAxis: [{ + axisLabel: { + formatter: function (value) { + return common.number.abbr(value, true); + } + } + }], + trans: true + }; + + var averageOption = { + animation: false, + tooltip: { + formatter: function (points) { + return points[0].name + "<br/>" + + $.map(points, function (point) { + return '<span style="display:inline-block;margin-right:5px;border-radius:10px;width:9px;height:9px;background-color:' + point.color + '"></span> ' + + point.seriesName + ": " + + common.number.abbr(point.value, false); + }).reverse().join("<br/>"); + } + }, + legend: { + x: 'center', y: 'bottom' + }, + areaStyle: {normal: {}}, + yAxis: [{ + axisLabel: { + formatter: function (value) { + return common.number.abbr(value, false); + } + } + }], + avg: true + }; + + function countStatus(site, status, groups, filed, limit) { + var jobCond = { + site: site, + status: status + }; + return SYSTEMMETRIC.aggSystemInstance(jobCond, groups, filed, limit); + } + + function aggGroup(site, groups, filed, limit) { + var jobCond = { + site: site + }; + return SYSTEMMETRIC.aggSystemInstance(jobCond, groups, filed, limit); + } + + function generateMetric(name, flag) { + var result = cache[name] || generateOvewviewMetric(name, flag); + return result; + } + + function generateOvewviewMetric(name, flag){ + var startTime = Time.startTime(); + var endTime = Time.endTime(); + var interval = Time.diffInterval(startTime, endTime); + var intervalMin = interval / 1000 / 60; + var trendStartTime = Time.align(startTime, interval); + var trendEndTime = Time.align(endTime, interval); + $scope.site = $wrapState.param.siteId; + var jobCond = { + site: $scope.site + }; + return SYSTEMMETRIC.aggMetricsToEntities(SYSTEMMETRIC.systemMetricsAggregation(jobCond, name, ["host","device"], "avg(value)", intervalMin, trendStartTime, trendEndTime), flag) + ._promise.then(function (list) { + var metricFlag = $.map(list, function (metrics) { + return metrics[0].flag; + }); + return [metricFlag, list]; + }); + } + + function mergeMetricToOneSeries(metricTitle, metrics, legendName, dataOption, option) { + var series = []; + $.each(metrics, function (i, metricMap) { + var mergeSeries = metricMap[0]; + $.each(metricMap, function (j, metriSeries) { + if(j > 0){ + $.each(metriSeries, function(n, dataNode){ + mergeSeries[n].value[0] = mergeSeries[n].value[0] * 1 + dataNode.value * 1; + }); + } + }); + if(dataOption.avg){ + $.each(mergeSeries, function(m, item) { + item.value[0] = item.value[0] / metricMap.length; + }); + } + series.push(SYSTEMMETRIC.metricsToSeries(legendName[i], mergeSeries, option, dataOption.trans)); + }); + return { + title: metricTitle, + series: series, + option: dataOption || {}, + loading: false + }; + } + + $scope.chartList = [ + { + name: "Memory Usage", + metrics: ["memfree"], + linename: ["memfree"], + option: digitalKbOption + }, + { + name: "Loadavg", + metrics: ["1minloadavg", "5minloadavg", "15minloadavg"], + linename: ["1minloadavg", "5minloadavg", "15minloadavg"], + option: digitalOption + }, + { + name: "Cpu Usage", + metrics: ["cputotalusage"], + linename: ["cputotalusage"], + option: averageOption + }, + { + name: "Network IO", + metrics: ["receivedbytes", "transmitbytes"], + linename: ["receive", "transmit"], + option: sizeoption + } + ]; + $scope.metricList = []; + $.each($scope.chartList, function (i) { + var chart = $scope.chartList[i]; + var chartname = chart.name; + $scope.metricList[chartname] = { + title: chartname, + series: {}, + option: {}, + loading: true, + promises: [] + }; + }); + + $scope.refresh = function () { + // status count + countStatus($scope.site, "active", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.systemactivenum = data.value[0]; + }); + }, function () { + $scope.systemactivenum = -1; + }); + + countStatus($scope.site, "warning", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.systemwarningnum = data.value[0]; + }); + }, function () { + $scope.systemwarningnum = -1; + }); + + countStatus($scope.site, "error", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.systemerrornum = data.value[0]; + }); + }, function () { + $scope.systemerrornum = -1; + }); + + // memoery count + aggGroup($scope.site, ["site"], "sum(totalMemoryMB),sum(usedMemoryMB),sum(totalDiskGB),sum(usedDiskGB)")._promise.then(function (res) { + $.map(res, function (data) { + $scope.totalMemoryMB = data.value[0]; + $scope.usedMemoryMB = data.value[1]; + $scope.totalDiskGB = data.value[2]; + $scope.usedDiskGB = data.value[3]; + }); + }, function () { + $scope.totalMemoryMB = -1; + $scope.usedMemoryMB = -1; + $scope.totalDiskGB = -1; + $scope.usedDiskGB = -1; + }); + + // metric chart + SYSTEMMETRIC.getMetricObj().then(function (res) { + var overviewMetricList = res.totaleverage; + console.log(overviewMetricList); + $.each($scope.chartList, function (i) { + var chart = $scope.chartList[i]; + var metricList = chart.metrics; + $.each(metricList, function (j) { + var metricKey = metricList[j]; + var metricspromies = generateMetric(overviewMetricList[metricKey], metricKey); + var chartname = chart.name; + $scope.metricList[chartname].promises.push(metricspromies); + }); + }); + + $.each($scope.chartList, function (k) { + var chart = $scope.chartList[k]; + var chartname = chart.name; + $q.all($scope.metricList[chartname].promises).then(function (resp) { + var series = []; + for (var r = 0; r < resp.length; r += 1) { + var rs = resp[r][1]; + if (typeof rs !== 'undefined' && rs.length > 0) { + series.push(rs); + } + } + $scope.metricList[chartname] = mergeMetricToOneSeries(chartname, series, chart.linename, chart.option); + }); + }); + }); + }; + + Time.onReload(function () { + cache = {}; + $.each($scope.chartList, function (i) { + var chart = $scope.chartList[i]; + var chartname = chart.name; + $scope.metricList[chartname] = { + title: chartname, + series: {}, + option: {}, + loading: true, + promises: [] + }; + }); + $scope.refresh(); + }, $scope); + $scope.refresh(); + }); + }); +})(); http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverDetailCtrl.js ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverDetailCtrl.js b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverDetailCtrl.js new file mode 100644 index 0000000..49f1dfa --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverDetailCtrl.js @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +(function () { + /** + * `register` without params will load the module which using require + */ + register(function (systemMetricApp) { + systemMetricApp.controller("serverDetailCtrl", function ($q, $wrapState, $scope, PageConfig, Time, SYSTEMMETRIC) { + var cache = {}; + $scope.site = $wrapState.param.siteId; + $scope.hostname = $wrapState.param.hostname; + PageConfig.title = 'System Node' + "(" + $scope.hostname + ")"; + $scope.metricList = []; + Time.autoRefresh = false; + + var sizeoption = { + animation: false, + tooltip: { + formatter: function (points) { + return points[0].name + "<br/>" + + $.map(points, function (point) { + return '<span style="display:inline-block;margin-right:5px;border-radius:10px;width:9px;height:9px;background-color:' + point.color + '"></span> ' + + point.seriesName + ": " + + common.number.abbr(point.value, true); + }).reverse().join("<br/>"); + } + }, + legend: { + x: 'center', y: 'bottom' + }, + areaStyle: {normal: {}}, + yAxis: [{ + axisLabel: { + formatter: function (value) { + return common.number.abbr(value, true); + } + } + }], + trans: true + }; + var digitalOption = { + animation: false, + tooltip: { + formatter: function (points) { + return points[0].name + "<br/>" + + $.map(points, function (point) { + return '<span style="display:inline-block;margin-right:5px;border-radius:10px;width:9px;height:9px;background-color:' + point.color + '"></span> ' + + point.seriesName + ": " + + common.number.abbr(point.value, false, 0); + }).reverse().join("<br/>"); + } + }, + yAxis: [{ + axisLabel: { + formatter: function (value) { + return common.number.abbr(value, false); + } + } + }] + }; + + function generateMetric(name, flag) { + var result = cache[name] || generateChartMetric(name, flag); + return result; + } + + function generateChartMetric(name, flag){ + var startTime = Time.startTime(); + var endTime = Time.endTime(); + var interval = Time.diffInterval(startTime, endTime); + var intervalMin = interval / 1000 / 60; + var trendStartTime = Time.align(startTime, interval); + var trendEndTime = Time.align(endTime, interval); + $scope.site = $wrapState.param.siteId; + var jobCond = { + site: $scope.site, + host: $scope.hostname + }; + return SYSTEMMETRIC.aggMetricsToEntities(SYSTEMMETRIC.systemMetricsAggregation(jobCond, name, ["device"], "avg(value)", intervalMin, trendStartTime, trendEndTime), flag) + ._promise.then(function (list) { + var metricFlag = $.map(list, function (metrics) { + return metrics[0].flag; + }); + return [metricFlag, list]; + }); + } + + function mergeMetricToOneSeries(metricTitle, metrics, legendName, dataOption, option) { + var series = []; + $.each(metrics, function (i, metricMap) { + if (typeof metricMap !== 'undefined') { + $.each(metricMap, function (j, metriSeries) { + var linename = []; + if(typeof metriSeries.group !== 'undefined'){ + linename.push(metriSeries.group); + } + linename.push(legendName[i]); + series.push(SYSTEMMETRIC.metricsToSeries(linename.join("."), metriSeries, option, dataOption.trans)); + }); + } + }); + return { + title: metricTitle, + series: series, + option: dataOption || {}, + loading: false + }; + } + + $scope.chartList = [ + { + name: "Cpu Usage", + metrics: ["cputotalusage"], + linename: ["usage"], + option: digitalOption + }, + { + name: "Memory Usage", + metrics: ["momoryusage"], + linename: ["usage"], + option: digitalOption + }, + { + name: "Network IO", + metrics: ["receivedbytes", "transmitbytes"], + linename: ["receive", "transmit"], + option: sizeoption + }, + { + name: "Loadavg", + metrics: ["1minloadavg", "5minloadavg", "15minloadavg"], + linename: ["1minloadavg", "5minloadavg", "15minloadavg"], + option: digitalOption + } + + ]; + $scope.metricList = []; + $.each($scope.chartList, function (i) { + var chart = $scope.chartList[i]; + var chartname = chart.name; + $scope.metricList[chartname] = { + title: chartname, + series: {}, + option: {}, + loading: true, + promises: [] + }; + }); + + $scope.refresh = function () { + var startTime = Time.startTime(); + var endTime = Time.endTime(); + var metricspromies = []; + + SYSTEMMETRIC.getMetricObj().then(function (res) { + var overviewMetricList = res.systemnode; + console.log(overviewMetricList); + $.each($scope.chartList, function (i) { + var chart = $scope.chartList[i]; + var metricList = chart.metrics; + $.each(metricList, function (j) { + var metricKey = metricList[j]; + var metricspromies = generateMetric(overviewMetricList[metricKey], metricKey); + var chartname = chart.name; + $scope.metricList[chartname].promises.push(metricspromies); + }); + }); + + $.each($scope.chartList, function (k) { + var chart = $scope.chartList[k]; + var chartname = chart.name; + $q.all($scope.metricList[chartname].promises).then(function (resp) { + var series = []; + for (var r = 0; r < resp.length; r += 1) { + var rs = resp[r][1]; + if (typeof rs !== 'undefined' && rs.length > 0) { + series.push(rs); + } + } + $scope.metricList[chartname] = mergeMetricToOneSeries(chartname, series, chart.linename, chart.option); + }); + }); + }); + + var condition = { + site: $scope.site, + role: "system", + hostname : $scope.hostname + }; + SYSTEMMETRIC.hostStatus(condition, 1)._promise.then(function (res) { + $scope.hostStatus = res; + }); + }; + Time.onReload(function () { + cache = {}; + $.each($scope.chartList, function (i) { + var chart = $scope.chartList[i]; + var chartname = chart.name; + $scope.metricList[chartname] = { + title: chartname, + series: {}, + option: {}, + loading: true, + promises: [] + }; + }); + $scope.refresh(); + }, $scope); + $scope.refresh(); + }); + }); +}) +(); http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverListCtrl.js ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverListCtrl.js b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverListCtrl.js new file mode 100644 index 0000000..c495612 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/ctrl/serverListCtrl.js @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +(function () { + /** + * `register` without params will load the module which using require + */ + register(function (systemMetricApp) { + + systemMetricApp.controller("serverListCtrl", function ($wrapState, $scope, PageConfig, SYSTEMMETRIC) { + + // Initialization + PageConfig.title = "System Server List"; + $scope.tableScope = {}; + $scope.site = $wrapState.param.siteId; + $scope.active = SYSTEMMETRIC.STATUS_ACTIVE; + $scope.warning = SYSTEMMETRIC.STATUS_WARNING; + $scope.error = SYSTEMMETRIC.STATUS_ERROR; + $scope.searchPathList = [["tags", "hostname"], ["tags", "rack"], ["tags", "site"], ["status"]]; + $scope.serverList = SYSTEMMETRIC.serverList($scope.site); + + }); + }); +})(); http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.html ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.html b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.html new file mode 100644 index 0000000..c21167e --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.html @@ -0,0 +1,6 @@ +<htm> + <body> + <h1>System Metric Monitor Application!</h1> + <i><b>url</b>: /apps/system</i> + </body> +</htm> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.js ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.js b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.js new file mode 100644 index 0000000..34c74ac --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/index.js @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +(function () { + /** + * `register` is global function that for application to set up 'controller', 'service', 'directive', 'route' in Eagle + */ + var systemMetricApp = register(['ngRoute', 'ngAnimate', 'ui.router', 'eagle.service']); + + systemMetricApp.route("systemMetric", { + url: "/hadoopService/system/overview?startTime&endTime", + site: true, + templateUrl: "partials/overview.html", + controller: "overviewCtrl", + resolve: {time: true} + }).route("serverList", { + url: "/hadoopService/system/serverList", + site: true, + templateUrl: "partials/serverList.html", + controller: "serverListCtrl" + }).route("serverDetail", { + url: "/hadoopService/system/serverDetail/:hostname", + site: true, + templateUrl: "partials/serverDetail.html", + controller: "serverDetailCtrl", + resolve: {time: true} + }); + + systemMetricApp.portal({ + name: "Services", icon: "heartbeat", list: [ + {name: "System", path: "hadoopService/system/overview"} + ] + }, true); + + systemMetricApp.service("SYSTEMMETRIC", function ($q, $http, Time, Site, Application) { + var SYSTEMMETRIC = window._SYSTEMMETRIC = {}; + SYSTEMMETRIC.STATUS_ACTIVE = "active"; + SYSTEMMETRIC.STATUS_WARNING = "warning"; + SYSTEMMETRIC.STATUS_ERROR = "error"; + SYSTEMMETRIC.QUERY_SYSTEM_METRICS = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]{*}&metricName=${metric}&pageSize=${limit}'; + SYSTEMMETRIC.QUERY_SYSTEM_METRICS_WITHTIME = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]{*}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}'; + SYSTEMMETRIC.QUERY_SYSTEM_METRICS_INTERVAL = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}&intervalmin=${intervalMin}&timeSeries=true'; + SYSTEMMETRIC.QUERY_SYSTEM_INSTANCE = '${baseURL}/rest/entities?query=SystemServiceInstance[${condition}]{*}&pageSize=${limit}'; + SYSTEMMETRIC.QUERY_SYSTEM_INSTANCE_AGG = "${baseURL}/rest/entities?query=SystemServiceInstance[${condition}]<${groups}>{${field}}&pageSize=${limit}"; + + var getQuery = SYSTEMMETRIC.getQuery = function (queryName, siteId) { + var baseURL; + siteId = siteId || Site.current().siteId; + var app = Application.find("SYSTEM_METRIC_WEB_APP", siteId)[0]; + var host = app.configuration["service.host"]; + var port = app.configuration["service.port"]; + + if (!host && !port) { + baseURL = ""; + } else { + if (host === "localhost" || !host) { + host = location.hostname; + } + if (!port) { + port = location.port; + } + baseURL = "http://"; + host + ":" + port; + } + + return common.template(SYSTEMMETRIC["QUERY_" + queryName], {baseURL: baseURL}); + }; + + function wrapList(promise) { + var _list = []; + _list._done = false; + _list._promise = promise.then( + /** + * @param {{}} res + * @param {{}} res.data + * @param {{}} res.data.obj + */ + function (res) { + _list.splice(0); + Array.prototype.push.apply(_list, res.data.obj); + _list._done = true; + return _list; + }); + return _list; + } + + function toFields(fields) { + return (fields || []).length > 0 ? $.map(fields, function (field) { + return "@" + field; + }).join(",") : "*"; + } + + SYSTEMMETRIC.metricsToSeries = function (name, metrics, option, transflag, rawData) { + if (arguments.length === 4 && typeof rawData === "object") { + option = rawData; + rawData = false; + } + var previous = 0; + data = $.map(metrics, function (metric, index) { + var temp = metric.value[0]; + if(transflag){ + metric.value[0] = metric.value[0] - previous; + } + previous = temp; + if(transflag && index === 0){ + return ; + } + return rawData ? metric.value[0] : { + x: metric.timestamp, + y: metric.value[0] + }; + }); + + return $.extend({ + name: name, + showSymbol: false, + type: "line", + data: data + }, option || {}); + }; + + SYSTEMMETRIC.get = function (url, params) { + return $http({ + url: url, + method: "GET", + params: params + }); + }; + + SYSTEMMETRIC.condition = function (condition) { + return $.map(condition, function (value, key) { + return "@" + key + '="' + value + '"'; + }).join(" AND "); + }; + + SYSTEMMETRIC.aggSystemInstance = function (condition, groups, field, limit) { + var fields = field.split(/\s*,\s*/); + var fieldStr = $.map(fields, function (field, index) { + var matches = field.match(/^([^\s]*)(\s+.*)?$/); + if (matches[2]) { + orderId = index; + } + return matches[1]; + }).join(", "); + var config = { + condition: SYSTEMMETRIC.condition(condition), + groups: toFields(groups), + field: fieldStr, + limit: limit || 10000 + }; + var metrics_url = common.template(getQuery("SYSTEM_INSTANCE_AGG"), config); + return wrapList(SYSTEMMETRIC.get(metrics_url)); + }; + + SYSTEMMETRIC.systemMetricsAggregation = function (condition, metric, groups, field, intervalMin, startTime, endTime, top, limit) { + var fields = field.split(/\s*,\s*/); + var orderId = -1; + var fieldStr = $.map(fields, function (field, index) { + var matches = field.match(/^([^\s]*)(\s+.*)?$/); + if (matches[2]) { + orderId = index; + } + return matches[1]; + }).join(", "); + + + var config = { + condition: SYSTEMMETRIC.condition(condition), + startTime: Time.format(startTime), + endTime: Time.format(endTime), + metric: metric, + groups: toFields(groups), + field: fieldStr, + order: orderId === -1 ? "" : ".{" + fields[orderId] + "}", + top: top ? "&top=" + top : "", + intervalMin: intervalMin, + limit: limit || 10000 + }; + + var metrics_url = common.template(getQuery("SYSTEM_METRICS_INTERVAL"), config); + var _list = wrapList(SYSTEMMETRIC.get(metrics_url)); + _list._aggInfo = { + groups: groups, + startTime: Time(startTime).valueOf(), + interval: intervalMin * 60 * 1000 + }; + _list._promise.then(function () { + _list.reverse(); + }); + return _list; + }; + + SYSTEMMETRIC.aggMetricsToEntities = function (list, param, flatten) { + var _list = []; + _list.done = false; + _list._promise = list._promise.then(function () { + var _startTime = list._aggInfo.startTime; + var _interval = list._aggInfo.interval; + $.each(list, function (i, obj) { + var tags = {}; + $.each(list._aggInfo.groups, function (j, group) { + tags[group] = obj.key[j]; + }); + var _subList = []; + _subList.group = obj.key.join(","); + $.each(obj.value[0], function (index, value) { + var node = { + timestamp: _startTime + index * _interval, + value: [value], + tags: tags, + flag: param + }; + _subList.push(node); + }); + /*var _subList = $.map(obj.value[0], function (value, index) { + return { + timestamp: _startTime + index * _interval, + value: [value], + tags: tags, + flag: param + }; + });*/ + if (flatten) { + _list.push.apply(_list, _subList); + } else { + _list.push(_subList); + } + }); + _list.done = true; + return _list; + }, function () { + return []; + }); + return _list; + }; + + SYSTEMMETRIC.hostStatus = function (condition, limit) { + var config = { + condition: SYSTEMMETRIC.condition(condition), + limit: limit || 10000 + }; + + var metrics_url = common.template(getQuery("SYSTEM_INSTANCE"), config); + return wrapList(SYSTEMMETRIC.get(metrics_url)); + }; + + + SYSTEMMETRIC.serverList = function (siteid, status, coresCount) { + var condition = { + site: siteid, + role: "system" + }; + if(typeof status !== 'undefined') { + condition.status = status; + } + if(typeof coresCount !== 'undefined') { + condition.cores = coresCount; + } + return SYSTEMMETRIC.hostStatus(condition); + }; + + SYSTEMMETRIC.getMetricObj = function () { + var deferred = $q.defer(); + $http.get("apps/system/config.json").success(function (resp) { + deferred.resolve(resp); + }); + return deferred.promise; + }; + + return SYSTEMMETRIC; + }); + + + systemMetricApp.requireCSS("style/index.css"); + systemMetricApp.require("widget/availabilityChart.js"); + systemMetricApp.require("ctrl/overview.js"); + systemMetricApp.require("ctrl/serverDetailCtrl.js"); + systemMetricApp.require("ctrl/serverListCtrl.js"); +})(); http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/overview.html ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/overview.html b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/overview.html new file mode 100644 index 0000000..de86079 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/overview.html @@ -0,0 +1,126 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<div class="box box-primary"> + <div class="box-header with-border"> + <span class="fa fa-television"></span> + <h3 class="box-title"> + System Summary Information + </h3> + </div> + <div class="box-body"> + <table class="table table-striped hadoopMetric-table"> + <tr> + <th width="20%"> + Server Node + </th> + <th width="80%"> + <span ng-show="!systemactivenum && !systemwarningnum && !systemerrornum">0</span> + <span ng-show="systemactivenum===-1 || systemwarningnum===-1 || systemerrornum===-1">N/A</span> + <a ui-sref="serverList({siteId: site})"> + <span ng-show="systemactivenum!==-1 && systemwarningnum!==-1 && systemerrornum!==-1">{{systemactivenum+systemwarningnum+systemerrornum}}</span> + </a> + <span class="label label-primary">Total</span> / + <a ui-sref="serverList({siteId: site, status:'active'})"> + <span ng-show="systemactivenum !== -1">{{systemactivenum || 0}}</span> + </a> + <span ng-show="systemactivenum === -1">N/A</span> + <span class="label label-success">Health</span> / + <a ui-sref="serverList({siteId: site, status:'warning'})"> + <span ng-show="systemwarningnum !== -1">{{systemwarningnum || 0}}</span> + </a> + <span ng-show="systemwarningnum === -1">N/A</span> + <span class="label label-warning">Warning</span> / + <a ui-sref="serverList({siteId: site, status:'error'})"> + <span ng-show="systemerrornum !== -1">{{systemerrornum || 0}}</span> + </a> + <span ng-show="systemerrornum === -1">N/A</span> + <span class="label label-danger">Unhealth</span> + </th> + </tr> + <tr> + <th>Cores</th> + <th> + <div ng-repeat="core in coresList"> + <a ui-sref="serverList({siteId: site, cores: core.key[0]})"> + <span>{{core.value[0]}}</span> + </a> + <span class="label label-primary">{{core.key[0]}} cores</span> + </div> + <span ng-show="!coresList">N/A</span> + </th> + </tr> + <tr> + <th>Disk Usage</th> + <th> + <span> + <span ng-show="usedDiskGB !== -1">{{usedDiskGB | number : 2 || 0}} GB / </span> + <span ng-show="!totalDiskGB === -1">N/A /</span> + <span ng-show="totalDiskGB !== -1">{{totalDiskGB | number : 2 || 0}} GB</span> + <span ng-show="!usedDiskGB === -1">N/A</span> + </span> + </th> + </tr> + <tr> + <th>Memory</th> + <th> + <span> + <span ng-show="usedMemoryMB !== -1">{{usedMemoryMB | number : 2 || 0}} MB / </span> + <span ng-show="!usedMemoryMB === -1">N/A /</span> + <span ng-show="totalMemoryMB !== -1">{{totalMemoryMB | number : 2 || 0}} MB</span> + <span ng-show="!totalMemoryMB === -1">N/A</span> + </span> + </th> + </tr> + </table> + </div> +</div> + +<div class="box box-primary"> + <div class="box-header with-border"> + <span class="fa fa-line-chart"></span> + <h3 class="box-title"> + Metrics + </h3> + </div> + <div class="box-body no-padding"> + <div class="row border-split"> + <div class="col-sm-6 col-md-4 col-lg-6" ng-repeat="chart in chartList track by $index"> + <div class="hadoopMetric-chart"> + <h3>{{metricList[chart.name].title}}</h3> + <div ng-show="metricList[chart.name].loading" chart class="hadoopMetric-chart-container"></div> + <div ng-show="metricList[chart.name].loading" class="overlay-wrapper"> + <div class="overlay"> + <i class="fa fa-refresh fa-spin"></i> + </div> + </div> + <div ng-show="metricList[chart.name].series.length" chart class="hadoopMetric-chart-container" + series="metricList[chart.name].series" + option="metricList[chart.name].option"></div> + <div ng-show="!metricList[chart.name].series.length && !metricList[chart.name].loading" class="hadoopMetric-chart-container"> + <div class="hadoopMetric-chart-container-withborder"> + <div class="hadoopMetric-no-chart-data"> + <span class="fa fa-question-circle"></span><span> NO DATA</span> + </div> + </div> + </div> + </div> + </div> + </div> + </div> +</div> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverDetail.html ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverDetail.html b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverDetail.html new file mode 100644 index 0000000..a3c7f47 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverDetail.html @@ -0,0 +1,111 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<div class="box box-primary"> + <div class="box-header with-border"> + <span class="fa fa-television"></span> + <h3 class="box-title"> + Summary + </h3> + </div> + <div class="box-body"> + <table class="table table-striped"> + <tbody ng-if="(hostStatus || []).length !== 0"> + <tr> + <th width="20%">Status</th> + <td width="80%" class="text-break"> + <span ng-if="hostStatus[0].status==='active'" class="label label-success">Healthy</span> + <span ng-if="hostStatus[0].status==='warning'" class="label label-warning">Warning</span> + <span ng-if="hostStatus[0].status==='error'" class="label label-danger">UnHealthy</span> + </td> + </tr> + <tr> + <th>Rack</th> + <td class="text-break">{{hostStatus[0].tags.rack}}</td> + </tr> + <tr> + <th>Cores</th> + <td class="text-break">{{hostStatus[0].cores}} </td> + </tr> + <tr> + <th>Disk Usage</th> + <td class="text-break">{{hostStatus[0].usedDiskGB | number : 2}} GB / {{hostStatus[0].totalDiskGB | number : 2}} GB</td> + </tr> + <tr> + <th>Memory Usage</th> + <td class="text-break">{{hostStatus[0].usedMemoryMB | number : 2}} MB / {{hostStatus[0].totalMemoryMB | number : 2}} MB</td> + </tr> + </tbody> + <tbody ng-if="hostStatus.length === 0"> + <tr> + <th>Status</th> + <td class="text-break"><span class="label label-danger">Unknow</span></td> + </tr> + <tr> + <th>Rack</th> + <td class="text-break"><span class="label label-danger">Unknow</span></td> + </tr> + <tr> + <th>Cores</th> + <td class="text-break"><span class="label label-danger">Unknow</span></td> + </tr> + <tr> + <th>Disk Usage</th> + <td class="text-break"><span class="label label-danger">Unknow</span></td> + </tr> + <tr> + <th>Memory Usage</th> + <td class="text-break"><span class="label label-danger">Unknow</span></td> + </tr> + </tbody> + </table> + </div> +</div> + +<div class="box box-primary"> + <div class="box-header with-border"> + <span class="fa fa-line-chart"></span> + <h3 class="box-title"> + Metrics + </h3> + </div> + <div class="box-body no-padding"> + <div class="row border-split"> + <div class="col-sm-6 hadoopMetric-col-md-6 col-lg-6" ng-repeat="chart in chartList track by $index"> + <div class="hadoopMetric-chart"> + <h3>{{metricList[chart.name].title}}</h3> + <div ng-show="metricList[chart.name].loading" class="hadoopMetric-chart-container"></div> + <div ng-show="metricList[chart.name].loading" class="overlay-wrapper"> + <div class="overlay"> + <i class="fa fa-refresh fa-spin"></i> + </div> + </div> + <div ng-show="metricList[chart.name].series.length" chart class="hadoopMetric-chart-container" + series="metricList[chart.name].series" + option="metricList[chart.name].option"></div> + <div ng-show="!metricList[chart.name].series.length && !metricList[chart.name].loading" class="hadoopMetric-chart-container"> + <div class="hadoopMetric-no-chart-data"> + <span class="fa fa-question-circle"></span><span> NO DATA</span> + </div> + </div> + </div> + </div> + </div> + </div> +</div> +</div> http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverList.html ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverList.html b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverList.html new file mode 100644 index 0000000..0f394ee --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/partials/serverList.html @@ -0,0 +1,60 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<div class="box box-primary"> + <div class="box-header with-border"> + <h3 class="box-title"> + <span ng-show="!serverList._done || isSorting" class="fa fa-refresh fa-spin no-animate"></span> + </h3> + </div> + <div class="box-body"> + <div id="serverList" sort-table="serverList" is-sorting="isSorting" + search-path-list="searchPathList" + scope="tableScope"> + <table class="table table-bordered"> + <thead> + <tr> + <th sortpath="tags.hostname">Host Name</th> + <th sortpath="tags.rack">Rack</th> + <th sortpath="tags.site">SiteId</th> + <th sortpath="status">Status</th> + <th sortpath="status">Cores</th> + <th sortpath="status">Disk Usage</th> + <th sortpath="status">Memory Usage</th> + </tr> + </thead> + <tbody> + <tr ng-repeat="item in serverList"> + <td> + <a ui-sref="serverDetail({siteId: site, hostname: item.tags.hostname})">{{item.tags.hostname}}</a> + </td> + <td>{{item.tags.rack}}</td> + <td>{{item.tags.site}}</td> + <td> + <span ng-if="item.status===active" class="label label-success">Healthy</span> + <span ng-if="item.status===warning" class="label label-warning">Warning</span> + <span ng-if="item.status===error" class="label label-danger">UnHealthy</span> + </td> + <td>{{item.cores}}</td> + <td>{{item.usedDiskGB | number : 2}} GB / {{item.totalDiskGB | number : 2}} GB</td> + <td>{{item.usedMemoryMB | number : 2}} MB / {{item.totalMemoryMB | number : 2}} MB</td> + </tr> + </tbody> + </table> + </div> + </div> +</div> http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/style/index.css ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/style/index.css b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/style/index.css new file mode 100644 index 0000000..aa215ab --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/style/index.css @@ -0,0 +1,88 @@ +@CHARSET "UTF-8"; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.hadoopMetric-chart { + position: relative; + margin-bottom: 15px; +} + +.hadoopMetric-chart h3 { + font-size: 20px; + margin: 10px 0 0 0; +} + +.hadoopMetric-chart .hadoopMetric-chart-container { + height: 300px; + position: relative; +} + +.hadoopMetric-chart .hadoopMetric-no-chart-data { + position:absolute; + left: 38%; + top: 40%; + font-size: 20px; + color: #999; +} + +.with-border .hadoopMetric-chart { + padding-bottom: 15px; + margin-bottom: 15px; + border-bottom: 1px solid #f4f4f4; +} + +.with-border .hadoopMetric-chart:last-child { + padding-bottom: 0; + margin-bottom: 0; + border-bottom: 0; +} + +.hadoopMetric-widget { + width: 100%; + box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1); + border-radius: 2px; + padding: 10px; +} + +.hadoopMetric-widget h3 { + margin: 0; + padding: 0; +} + +.hadoopMetric-widget .hadoopMetric-chart-container { + height: 100px; +} + +.hadoopMetric-summary-a { + color: #fff; +} + +.small-box.hadoopMetric-widget { + margin: 0; + height: 100%; + min-height: 110px; +} + +.hadoopMetric-widget-detail { + margin-left: 12px; + margin-top: 10px; +} + +.hadoopMetric-table { + text-align: right; +} http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/widget/availabilityChart.js ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/widget/availabilityChart.js b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/widget/availabilityChart.js new file mode 100644 index 0000000..2529f03 --- /dev/null +++ b/eagle-metric/eagle-system-web/src/main/webapp/app/apps/system/widget/availabilityChart.js @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +(function () { + /** + * `register` without params will load the module which using require + */ + register(function (systemMetricApp) { + systemMetricApp.directive("systemMetricWidget", function () { + return { + restrict: 'AE', + controller: function ($scope, $attrs, SYSTEMMETRIC, Application, $interval, Site, $wrapState) { + // Get site + var site = $scope.site; + var refreshInterval; + + if(!site) { + $scope.list = $.map(Application.find("SYSTEM_METRIC_WEB_APP"), function (app) { + return { + siteId: app.site.siteId, + siteName: app.site.siteName || app.site.siteId + }; + }); + } else { + $scope.list = [{ + siteId: site.siteId, + siteName: site.siteName || site.siteId + }]; + } + // Get type + $scope.type = $attrs.type; + + // Customize chart color + $scope.bgColor = "yellow-active"; + + function countStatus(site, status, groups, filed, limit) { + var jobCond = { + site: site, + status: status + }; + return SYSTEMMETRIC.aggSystemInstance(jobCond, groups, filed, limit); + } + + // Ref: jpm widget if need keep refresh the widget + + function refresh() { + $.each($scope.list, function (i, site) { + // status count + countStatus(site.siteId, "active", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.systemactivenum = data.value[0]; + }); + }, function () { + $scope.systemactivenum = -1; + }); + + countStatus(site.siteId, "warning", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.systemwarningnum = data.value[0]; + }); + }, function () { + $scope.systemwarningnum = -1; + }); + + countStatus(site.siteId, "error", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.systemerrornum = data.value[0]; + }); + }, function () { + $scope.systemerrornum = -1; + }); + }); + } + + refresh(); + refreshInterval = $interval(refresh, 30 * 1000); + + $scope.$on('$destroy', function () { + $interval.cancel(refreshInterval); + }); + }, + template: + '<div class="small-box hadoopMetric-widget bg-{{bgColor}}">' + + '<div class="inner">' + + '<h3>{{type}}</h3>' + + '<div ng-show="systemactivenum!==-1 && systemwarningnum!==-1 && systemerrornum!==-1" class="hadoopMetric-widget-detail">' + + '<a ui-sref="serverList({siteId: site.siteId})">' + + '<span>{{systemactivenum+systemwarningnum+systemerrornum || 0}}</span> Node (' + + '<span>{{systemactivenum || 0}}</span> Healthy / ' + + '<span>{{systemwarningnum || 0}}</span> Warning / ' + + '<span>{{systemerrornum || 0}}</span> Unhealthy)' + + '</a>' + + '</div>' + + '<div ng-show="systemactivenum===-1 || systemwarningnum===-1 || systemerrornum===-1" class="hadoopMetric-widget-detail">' + + '<span>N/A</span> Node (' + + '<span>N/A</span> Healthy / ' + + '<span>N/A</span> Warning / ' + + '<span>N/A</span> Unhealthy)' + + '</div>' + + '</div>' + + '<div class="icon">' + + '<i class="fa fa-taxi"></i>' + + '</div>' + + '</div>', + replace: true + }; + }); + + function withType(serviceType) { + /** + * Customize the widget content. Return false will prevent auto compile. + * @param {{}} $element + * @param {function} $element.append + */ + return function registerWidget($element) { + $element.append( + $("<div system-metric-widget data-type='" + serviceType + "'>") + ); + }; + } + systemMetricApp.widget("availabilitySystemChart", withType('System'), true); + }); +})(); http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/eagle-system-web/src/main/webapp/package.json ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-system-web/src/main/webapp/package.json b/eagle-metric/eagle-system-web/src/main/webapp/package.json new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-metric/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-metric/pom.xml b/eagle-metric/pom.xml index 6bdf434..a7f8dfd 100644 --- a/eagle-metric/pom.xml +++ b/eagle-metric/pom.xml @@ -24,12 +24,13 @@ <modelVersion>4.0.0</modelVersion> <artifactId>eagle-metric-parent</artifactId> - <name>Eagle::App::HadoopMetric</name> + <name>Eagle::App::HadoopMetric::Parent</name> <packaging>pom</packaging> <modules> <module>eagle-hadoop-metric</module> <module>eagle-hdfs-web</module> <module>eagle-hbase-web</module> + <module>eagle-system-web</module> </modules> <dependencies> http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-server/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml index 8db44cd..7df204c 100644 --- a/eagle-server/pom.xml +++ b/eagle-server/pom.xml @@ -351,12 +351,17 @@ <version>${project.version}</version> </dependency> + <!-- add metirc webUI --> <dependency> <groupId>org.apache.eagle</groupId> <artifactId>eagle-hadoop-metric</artifactId> <version>${project.version}</version> </dependency> - + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-hbase-web</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.eagle</groupId> <artifactId>eagle-hdfs-web</artifactId> @@ -364,9 +369,10 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-hbase-web</artifactId> + <artifactId>eagle-system-web</artifactId> <version>${project.version}</version> </dependency> + </dependencies> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider ---------------------------------------------------------------------- diff --git a/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider index 54650bd..e6a3723 100644 --- a/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider +++ b/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider @@ -32,4 +32,5 @@ org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider ## Metric org.apache.eagle.metric.HdfsMetricWebApplicationProvider -org.apache.eagle.metric.HBaseMetricWebApplicationProvider \ No newline at end of file +org.apache.eagle.metric.HBaseMetricWebApplicationProvider +org.apache.eagle.metric.SystemMetricWebApplicationProvider \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-topology-check/eagle-topology-app/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/pom.xml b/eagle-topology-check/eagle-topology-app/pom.xml index dc74b9f..55cb525 100644 --- a/eagle-topology-check/eagle-topology-app/pom.xml +++ b/eagle-topology-check/eagle-topology-app/pom.xml @@ -47,6 +47,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-data-process</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>org.json</groupId> <artifactId>json</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java index 87ff27a..df14bfa 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java @@ -24,6 +24,8 @@ import com.typesafe.config.Config; import org.apache.eagle.app.StormApplication; import org.apache.eagle.app.environment.impl.StormEnvironment; import org.apache.eagle.app.messaging.StormStreamSink; +import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSpoutProvider; +import org.apache.eagle.topology.extractor.system.SystemCheckPersistBolt; import org.apache.eagle.topology.storm.HealthCheckParseBolt; import org.apache.eagle.topology.storm.TopologyCheckAppSpout; import org.apache.eagle.topology.storm.TopologyDataPersistBolt; @@ -31,12 +33,15 @@ import org.apache.eagle.topology.storm.TopologyDataPersistBolt; public class TopologyCheckApp extends StormApplication { private static final String TOPOLOGY_HEALTH_CHECK_STREAM = "topology_health_check_stream"; + private static final String SYSTEM_COLLECTOR_CONFIG_PREFIX = "dataSourceConfig.system"; @Override public StormTopology execute(Config config, StormEnvironment environment) { TopologyCheckAppConfig topologyCheckAppConfig = TopologyCheckAppConfig.newInstance(config); String spoutName = TopologyCheckAppConfig.TOPOLOGY_DATA_FETCH_SPOUT_NAME; + String systemSpoutName = TopologyCheckAppConfig.SYSTEM_DATA_FETCH_SPOUT_NAME; + String systemPersistBoltName = TopologyCheckAppConfig.SYSTEM_ENTITY_PERSIST_BOLT_NAME; String persistBoltName = TopologyCheckAppConfig.TOPOLOGY_ENTITY_PERSIST_BOLT_NAME; String parseBoltName = TopologyCheckAppConfig.PARSE_BOLT_NAME; String kafkaSinkBoltName = TopologyCheckAppConfig.SINK_BOLT_NAME; @@ -62,11 +67,27 @@ public class TopologyCheckApp extends StormApplication { StormStreamSink<?> sinkBolt = environment.getStreamSink(TOPOLOGY_HEALTH_CHECK_STREAM, config); topologyBuilder.setBolt( - kafkaSinkBoltName, - sinkBolt, - topologyCheckAppConfig.dataExtractorConfig.numKafkaSinkBolt + kafkaSinkBoltName, + sinkBolt, + topologyCheckAppConfig.dataExtractorConfig.numKafkaSinkBolt ).setNumTasks(topologyCheckAppConfig.dataExtractorConfig.numKafkaSinkBolt).shuffleGrouping(parseBoltName); + // system check data collector + if (topologyCheckAppConfig.systemConfig.systemInstanceEnable) { + topologyBuilder.setSpout( + systemSpoutName, + new KafkaSpoutProvider(SYSTEM_COLLECTOR_CONFIG_PREFIX).getSpout(config), + topologyCheckAppConfig.dataExtractorConfig.numDataFetcherSpout + ).setNumTasks(topologyCheckAppConfig.dataExtractorConfig.numDataFetcherSpout); + + // system check data persist + topologyBuilder.setBolt( + systemPersistBoltName, + new SystemCheckPersistBolt(topologyCheckAppConfig), + topologyCheckAppConfig.dataExtractorConfig.numEntityPersistBolt + ).setNumTasks(topologyCheckAppConfig.dataExtractorConfig.numEntityPersistBolt).shuffleGrouping(systemSpoutName); + } + return topologyBuilder.createTopology(); } } http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java index 7a339a6..b67867b 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppConfig.java @@ -31,6 +31,8 @@ import java.util.List; public class TopologyCheckAppConfig implements Serializable { public static final String TOPOLOGY_DATA_FETCH_SPOUT_NAME = "topologyDataFetcherSpout"; + public static final String SYSTEM_DATA_FETCH_SPOUT_NAME = "systemDataFetcherSpout"; + public static final String SYSTEM_ENTITY_PERSIST_BOLT_NAME = "systemEntityPersistBolt"; public static final String TOPOLOGY_ENTITY_PERSIST_BOLT_NAME = "topologyEntityPersistBolt"; public static final String PARSE_BOLT_NAME = "parserBolt"; public static final String SINK_BOLT_NAME = "sinkBolt"; @@ -44,6 +46,7 @@ public class TopologyCheckAppConfig implements Serializable { public HBaseConfig hBaseConfig; public HdfsConfig hdfsConfig; public MRConfig mrConfig; + public SystemConfig systemConfig; public List<TopologyConstants.TopologyType> topologyTypes; private Config config; @@ -113,6 +116,15 @@ public class TopologyCheckAppConfig implements Serializable { hdfsConfig = new HdfsConfig(); hdfsConfig.namenodeUrls = config.getString("dataSourceConfig.hdfs.namenodeUrl").split(",\\s*"); } + + if (config.hasPath("dataSourceConfig.system.enabled") && config.getBoolean("dataSourceConfig.system.enabled")) { + systemConfig = new SystemConfig(); + systemConfig.systemInstanceKafkaTopic = getOptionalConfig("dataSourceConfig.system.topic", null); + systemConfig.systemInstanceKafkaSchemeCls = getOptionalConfig("dataSourceConfig.system.schemeCls", null); + systemConfig.systemInstanceZkQuorum = getOptionalConfig("dataSourceConfig.system.zkConnection", null); + systemConfig.systemInstanceSendBatchSize = config.getInt("dataSourceConfig.system.dataSendBatchSize"); + systemConfig.systemInstanceEnable = config.getBoolean("dataSourceConfig.system.enabled"); + } } public static class DataExtractorConfig implements Serializable { @@ -144,6 +156,14 @@ public class TopologyCheckAppConfig implements Serializable { public String[] namenodeUrls; } + public static class SystemConfig implements Serializable { + public String systemInstanceKafkaTopic; + public String systemInstanceZkQuorum; + public String systemInstanceKafkaSchemeCls; + public int systemInstanceSendBatchSize; + public boolean systemInstanceEnable; + } + private String getOptionalConfig(String key, String defaultValue) { if (this.config.hasPath(key)) { return config.getString(key); http://git-wip-us.apache.org/repos/asf/eagle/blob/877d3e77/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/system/SystemCheckPersistBolt.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/system/SystemCheckPersistBolt.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/system/SystemCheckPersistBolt.java new file mode 100644 index 0000000..255f7ca --- /dev/null +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/system/SystemCheckPersistBolt.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.topology.extractor.system; + +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichBolt; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; +import org.apache.eagle.service.client.IEagleServiceClient; +import org.apache.eagle.service.client.impl.BatchSender; +import org.apache.eagle.service.client.impl.EagleServiceClientImpl; +import org.apache.eagle.topology.TopologyCheckAppConfig; +import org.apache.eagle.topology.TopologyConstants; +import org.apache.eagle.topology.entity.SystemServiceTopologyAPIEntity; +import org.apache.eagle.topology.resolver.TopologyRackResolver; +import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.eagle.topology.TopologyConstants.*; + +/** + * system check persist bolt. + */ +public class SystemCheckPersistBolt extends BaseRichBolt { + + private static final Logger LOG = LoggerFactory.getLogger(SystemCheckPersistBolt.class); + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + private final int batchSize; + private OutputCollector collector; + private TopologyCheckAppConfig config; + private IEagleServiceClient client; + private BatchSender batchSender; + private TopologyRackResolver rackResolver; + private String site; + + public SystemCheckPersistBolt(TopologyCheckAppConfig config) { + this.config = config; + this.site = config.dataExtractorConfig.site; + this.batchSize = config.systemConfig.systemInstanceSendBatchSize > 0 ? config.systemConfig.systemInstanceSendBatchSize : 1; + } + + @Override + public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { + this.collector = outputCollector; + this.client = new EagleServiceClientImpl(config.getConfig()); + if (this.batchSize > 0) { + this.batchSender = client.batch(this.batchSize); + } + this.rackResolver = new DefaultTopologyRackResolver(); + if (config.dataExtractorConfig.resolverCls != null) { + try { + rackResolver = config.dataExtractorConfig.resolverCls.newInstance(); + rackResolver.prepare(config); + } catch (InstantiationException | IllegalAccessException e) { + LOG.error(e.getMessage(), e); + } + } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields(TopologyConstants.SERVICE_NAME_FIELD, TopologyConstants.TOPOLOGY_DATA_FIELD)); + } + + @Override + public void execute(Tuple tuple) { + long excuteTimestamp = System.currentTimeMillis(); + if (tuple == null) { + return; + } + // transform tuple to SystemServiceTopologyAPIEntity + String metrix = tuple.getString(0); + try { + SystemServiceTopologyAPIEntity entity = createEntity(metrix, excuteTimestamp); + if (batchSize <= 1) { + GenericServiceAPIResponseEntity<String> response = this.client.create(Collections.singletonList(entity)); + if (!response.isSuccess()) { + LOG.error("Service side error: {}", response.getException()); + collector.reportError(new IllegalStateException(response.getException())); + } + } else { + batchSender.send(entity); + } + this.collector.ack(tuple); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + this.collector.fail(tuple); + } + + } + + private SystemServiceTopologyAPIEntity createEntity(String metrix, long excuteTimestamp) throws IOException { + SystemServiceTopologyAPIEntity entity = mapper.readValue(metrix, SystemServiceTopologyAPIEntity.class); + Map<String, String> tags = new HashMap<String, String>(); + entity.setTags(tags); + tags.put(SITE_TAG, this.site); + tags.put(ROLE_TAG, TopologyConstants.SYSTEM_ROLE); + tags.put(HOSTNAME_TAG, entity.getHost()); + String rack = rackResolver.resolve(entity.getHost()); + tags.put(RACK_TAG, rack); + entity.setLastUpdateTime(excuteTimestamp); + entity.setTimestamp(excuteTimestamp); + // status + entity.setStatus("active"); + return entity; + } +}
