Repository: eagle Updated Branches: refs/heads/master 7d39863f4 -> af9b056dc
[EAGLE-1016] fix keyword "group" conflict in JDBC storage https://issues.apache.org/jira/browse/EAGLE-1016 If you meet the same issue after applying this patch, please drop the old table first. Author: Zhao, Qingwen <[email protected]> Closes #933 from qingwen220/EAGLE-1016. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/af9b056d Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/af9b056d Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/af9b056d Branch: refs/heads/master Commit: af9b056dc453f46c6a2aeb92d8dd6fe503f1b577 Parents: 7d39863 Author: Zhao, Qingwen <[email protected]> Authored: Fri May 19 13:55:27 2017 +0800 Committer: Zhao, Qingwen <[email protected]> Committed: Fri May 19 13:55:27 2017 +0800 ---------------------------------------------------------------------- eagle-assembly/src/main/conf/eagle.conf | 26 +++++++++++--------- .../metadata/model/MetricSchemaEntity.java | 6 ++--- .../jdbc/conn/ConnectionConfigFactory.java | 22 +++++++++++------ .../system_metric_collector.py | 2 +- .../scheduler/QueueStructureAPIEntity.java | 1 + .../model/scheduler/RunningQueueAPIEntity.java | 1 + .../mr/historyentity/JobExecutionAPIEntity.java | 1 + .../eagle/metric/HadoopMetricMonitorApp.java | 5 ++-- .../app/dev/public/js/ctrls/metricCtrl.js | 2 +- .../entity/HBaseServiceTopologyAPIEntity.java | 1 + .../entity/HdfsServiceTopologyAPIEntity.java | 2 +- .../entity/MRServiceTopologyAPIEntity.java | 1 + .../entity/SystemServiceTopologyAPIEntity.java | 1 + 13 files changed, 43 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-assembly/src/main/conf/eagle.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/eagle.conf b/eagle-assembly/src/main/conf/eagle.conf index 496e3d0..3441fe6 100644 --- a/eagle-assembly/src/main/conf/eagle.conf +++ b/eagle-assembly/src/main/conf/eagle.conf @@ -40,32 +40,34 @@ zookeeper { storage { # storage type: ["hbase","jdbc"] - # default is "hbase" + # "hbase" is preferred type = "hbase" + ## for prod env hbase { autoCreateTable = true - # hbase configuration: hbase.zookeeper.quorum - # default is "localhost" zookeeperQuorum = "localhost" - # hbase configuration: hbase.zookeeper.property.clientPort - # default is 2181 zookeeperPropertyClientPort = 2181 - # hbase configuration: zookeeper.znode.parent - # default is "/hbase" zookeeperZnodeParent = "/hbase-unsecure" - - # eagle web login profile: [sandbox, default] - # default is sandbox tableNamePrefixedWithEnvironment = false - # eagle coprocessor enabled or not: [true, false] - # default is false coprocessorEnabled = false } + + ## for dev env, in which JPM is NOT supported + jdbc { + adapter = "mysql" + username = "root" + password = "" + database = "eagle" + driverClass = "com.mysql.jdbc.Driver" + connectionUrl = "jdbc:mysql://localhost:3306/eagle" + connectionProps = "encoding=UTF-8" + connectionMax = 8 + } } # --------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java index 3bd5825..36eef9d 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java +++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java @@ -30,12 +30,12 @@ import java.util.List; @Service(MetricSchemaEntity.METRIC_SCHEMA_SERVICE) @JsonIgnoreProperties(ignoreUnknown = true) @TimeSeries(false) -@Tags({"site","site","group"}) +@Tags({"site","metricName","metricGroup"}) public class MetricSchemaEntity extends TaggedLogAPIEntity { static final String METRIC_SCHEMA_SERVICE = "MetricSchemaService"; - public static final String METRIC_NAME_TAG = "name"; public static final String METRIC_SITE_TAG = "site"; - public static final String METRIC_GROUP_TAG = "group"; + public static final String METRIC_NAME_TAG = "metricName"; + public static final String METRIC_GROUP_TAG = "metricGroup"; @Column("a") private List<String> dimensionFields; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java index 71f19aa..bb9e5f1 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java @@ -32,21 +32,29 @@ public class ConnectionConfigFactory { String username = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DB_USERNAME); String password = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DB_PASSWORD); String connUrl = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_URL); - String connProps = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_PROPS); String adapter = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_ADAPTER); String databaseName = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DATABASE); String driverClass = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DRIVER_CLASS); - String connMaxSize = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_MAX_SIZE); + + + String connProps = null; + if (EagleConfigFactory.load().getConfig().hasPath(JdbcConstants.EAGLE_CONN_PROPS)) { + connProps = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_PROPS); + } + + String connMaxSize = null; + if (EagleConfigFactory.load().getConfig().hasPath(JdbcConstants.EAGLE_CONN_MAX_SIZE)) { + connMaxSize = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_MAX_SIZE); + } ConnectionConfig config = new ConnectionConfig(); if(username != null) config.setUserName(username); if(password != null)config.setPassword(password); if(connUrl != null) config.setConnectionUrl(connUrl); - if(connProps!=null) config.setConnectionProperties(connProps); - if(adapter!=null) config.setAdapter(adapter); - if(databaseName!=null) config.setDatabaseName(databaseName); - if(driverClass!=null) config.setDriverClassName(driverClass); - + if(connProps != null) config.setConnectionProperties(connProps); + if(adapter != null) config.setAdapter(adapter); + if(databaseName != null) config.setDatabaseName(databaseName); + if(driverClass != null) config.setDriverClassName(driverClass); if(connMaxSize!=null){ config.setConnectionMaxActive(Integer.parseInt(connMaxSize)); http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-external/hadoop_jmx_collector/system_metric_collector.py ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/system_metric_collector.py b/eagle-external/hadoop_jmx_collector/system_metric_collector.py index fba06f2..501798b 100644 --- a/eagle-external/hadoop_jmx_collector/system_metric_collector.py +++ b/eagle-external/hadoop_jmx_collector/system_metric_collector.py @@ -224,7 +224,7 @@ class SystemMetricCollector(MetricCollector): output = os.popen('sudo ipmitool sdr | grep Temp | grep CPU').readlines() for item in output: items = re.split("^(CPU\d+)\sTemp\.\s+\|\s+(\d+|\d+\.\d+)\s", item.rstrip()) - event = self.new_metric("System.CPU") + event = self.new_metric("system.cpu") event["timestamp"] = int(round(time.time() * 1000)) event["metric"] = DATA_TYPE + "." + 'cpu.temp' event["value"] = items[2] http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java index 72f67bc..fde709d 100644 --- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java +++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java @@ -31,6 +31,7 @@ import java.util.List; @Service(HadoopClusterConstants.QUEUE_MAPPING_SERVICE_NAME) @TimeSeries(false) @Partition( {"site"}) +@Tags({HadoopClusterConstants.TAG_SITE, HadoopClusterConstants.TAG_QUEUE, HadoopClusterConstants.TAG_PARENT_QUEUE}) public class QueueStructureAPIEntity extends TaggedLogAPIEntity { @Column("a") private List<String> subQueues; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java index be4ef31..a93bad0 100755 --- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java +++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java @@ -32,6 +32,7 @@ import java.util.List; @Service(HadoopClusterConstants.RUNNING_QUEUE_SERVICE_NAME) @TimeSeries(true) @Partition( {"site"}) +@Tags({HadoopClusterConstants.TAG_SITE, HadoopClusterConstants.TAG_QUEUE, HadoopClusterConstants.TAG_PARENT_QUEUE}) public class RunningQueueAPIEntity extends TaggedLogAPIEntity { @Column("a") private String state; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java index 746843d..7a649b7 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = true), @Index(name = "Index_2_jobDefId", columns = { "jobDefId" }, unique = false) }) +@Tags({"site", "jobId", "jobName", "jobDefId", "jobType", "user", "queue"}) public class JobExecutionAPIEntity extends JobBaseAPIEntity { @Column("a") private String currentState; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java ---------------------------------------------------------------------- diff --git a/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java b/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java index aa1753f..1761aba 100644 --- a/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java +++ b/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java @@ -51,15 +51,14 @@ public class HadoopMetricMonitorApp extends StormApplication { .siteAs(AppConfigUtils.getSiteId(config)) .namedByField("metric") .eventTimeByField("timestamp") - .dimensionFields("host", "group", "site", "device") + .dimensionFields("host", "site", "device") .granularity(Calendar.SECOND) .valueField("value"); return environment.newApp(config) .fromStream("HADOOP_JMX_METRIC_STREAM").transformBy(new CounterToRateFunction(hadoopMetricDescriptor,3, TimeUnit.SECONDS, ClockWithOffset.INSTANCE)) .saveAsMetric(hadoopMetricDescriptor) .fromStream("SYSTEM_METRIC_STREAM").transformBy(new CounterToRateFunction(systemMetricDescriptor,3, TimeUnit.SECONDS, ClockWithOffset.INSTANCE)) - .saveAsMetric(systemMetricDescriptor - ) + .saveAsMetric(systemMetricDescriptor) .toTopology(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js ---------------------------------------------------------------------- diff --git a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js index b310738..e0f0f15 100644 --- a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js +++ b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js @@ -40,7 +40,7 @@ $scope.metricList = [$scope.metricName]; CompatibleEntity.groups({ query: 'MetricSchemaService', - groups: 'name', + groups: 'metricName', fields: 'count', limit: 9999, })._promise.then(function (res) { http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java index 9d7233e..ce829ed 100644 --- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java +++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Prefix("hbaseservicestatus") @Service(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME) @TimeSeries(false) +@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG}) public class HBaseServiceTopologyAPIEntity extends TopologyBaseAPIEntity { @Column("a") private String status; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java index 4054cba..1362736 100644 --- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java +++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java @@ -22,13 +22,13 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.eagle.log.entity.meta.*; import org.apache.eagle.topology.TopologyConstants; - @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("hadoop_topology") @ColumnFamily("f") @Prefix("hdfsservicestatus") @Service(TopologyConstants.HDFS_INSTANCE_SERVICE_NAME) @TimeSeries(false) +@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG}) public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity { @Column("a") private String status; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java index fa7e1a4..4c3040e 100644 --- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java +++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Prefix("mrservicestatus") @Service(TopologyConstants.MR_INSTANCE_SERVICE_NAME) @TimeSeries(false) +@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG}) public class MRServiceTopologyAPIEntity extends TopologyBaseAPIEntity { @Column("a") private String status; http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java index 119cb46..e441741 100644 --- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java +++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java @@ -29,6 +29,7 @@ import org.apache.eagle.topology.TopologyConstants; @Prefix("systemservicestatus") @Service(TopologyConstants.SYSTEM_INSTANCE_SERVICE_NAME) @TimeSeries(false) +@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG}) public class SystemServiceTopologyAPIEntity extends TopologyBaseAPIEntity { @Column("a") private String status;
