incubator-eagle git commit: [EAGLE-844] Fix a potential NPE

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master a34ef878a -> 73642a4ee


[EAGLE-844] Fix a potential NPE

https://issues.apache.org/jira/browse/EAGLE-844

Author: lifove 
Author: JC 

Closes #751 from lifove/master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/73642a4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/73642a4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/73642a4e

Branch: refs/heads/master
Commit: 73642a4ee2e307a572457bca664b790385038c81
Parents: a34ef87
Author: lifove 
Authored: Tue Dec 20 19:16:25 2016 +0800
Committer: wujinhu 
Committed: Tue Dec 20 19:16:25 2016 +0800

--
 .../org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/73642a4e/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
--
diff --git 
a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
 
b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
index 354d641..8e4005c 100644
--- 
a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
+++ 
b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
@@ -65,7 +65,7 @@ public class HBaseEntitySchemaManager {
 try {
 admin = new HBaseAdmin(conf);
 Map entityServiceMap = 
EntityDefinitionManager.entities();
-if (entityServiceMap != null || entityServiceMap.values() != null) 
{
+if (entityServiceMap != null) {
 for (EntityDefinition entityDefinition : 
entityServiceMap.values()) {
 createTable(entityDefinition);
 }

incubator-eagle git commit: [MINOR] add stream data source config for mr history job

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master e24de5c7e -> 229d7b907


[MINOR] add stream data source config for mr history job

Author: wujinhu 

Closes #748 from wujinhu/EAGLE-796.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/229d7b90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/229d7b90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/229d7b90

Branch: refs/heads/master
Commit: 229d7b9073a430de7dbbb12d39ddc330646eb458
Parents: e24de5c
Author: wujinhu 
Authored: Fri Dec 16 14:55:47 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 16 14:55:47 2016 +0800

--
 .../apache/eagle/jpm/mr/history/storm/JobHistorySpout.java  | 9 ++---
 ...eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml | 9 -
 2 files changed, 14 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/229d7b90/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
index 0cd30ae..d7daa5e 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
@@ -140,7 +140,12 @@ public class JobHistorySpout extends BaseRichSpout {
 JobHistoryZKStateManager.instance().init(appConfig.getZkStateConfig());
 JobHistoryZKStateManager.instance().ensureJobPartition(partitionId, 
numTotalPartitions);
 interceptor.setSpoutOutputCollector(collector);
-
+if (streamPublishers != null) {
+for (StreamPublisher streamPublisher : streamPublishers) {
+streamPublisher.setCollector(this.interceptor);
+
StreamPublisherManager.getInstance().addStreamPublisher(streamPublisher);
+}
+}
 try {
 jhfLCM = new JobHistoryDAOImpl(jobHistoryEndpointConfig);
 driver = new JHFCrawlerDriverImpl(
@@ -189,8 +194,6 @@ public class JobHistorySpout extends BaseRichSpout {
 if (streamPublishers != null) {
 for (StreamPublisher streamPublisher : streamPublishers) {
 declarer.declareStream(streamPublisher.stormStreamId(), new 
Fields("f1", "message"));
-streamPublisher.setCollector(this.interceptor);
-
StreamPublisherManager.getInstance().addStreamPublisher(streamPublisher);
 }
 } else {
 declarer.declare(new Fields("f1", "message"));

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/229d7b90/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index ccf3c6b..1ff9b85 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -96,6 +96,13 @@
 true
 
 
+dataSourceConfig.zkConnection
+Kafka Zookeeper Quorum
+localhost:2181
+kafka zookeeper connection
+true
+
+
 dataSinkConfig.serializerClass
 Serializer Class For Kafka Message Value
 kafka.serializer.StringEncoder
@@ -221,7 +228,7 @@
 string
 
 
-queue
+hostname
 string
 
 

incubator-eagle git commit: [MINOR] fix job & task attempt stream publisher bug in storm env

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8fe968cb7 -> e24de5c7e


[MINOR] fix job & task attempt stream publisher bug in storm env

Author: wujinhu 

Closes #747 from wujinhu/EAGLE-796.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e24de5c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e24de5c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e24de5c7

Branch: refs/heads/master
Commit: e24de5c7e85f7dba7eb5dfc4aee56b421c0e8ec4
Parents: 8fe968c
Author: wujinhu 
Authored: Fri Dec 16 11:23:40 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 16 11:23:40 2016 +0800

--
 .../jpm/mr/history/MRHistoryJobApplication.java | 15 +---
 .../crawler/DefaultJHFInputStreamCallback.java  |  6 ++---
 .../jpm/mr/history/parser/JHFParserFactory.java |  3 +--
 .../JobEntityCreationEagleServiceListener.java  |  8 +++
 .../history/publisher/JobStreamPublisher.java   |  4 ++--
 .../mr/history/publisher/StreamPublisher.java   | 13 +--
 .../publisher/TaskAttemptStreamPublisher.java   |  4 ++--
 .../jpm/mr/history/storm/JobHistorySpout.java   | 24 
 history.MRHistoryJobApplicationProvider.xml | 15 
 9 files changed, 53 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e24de5c7/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
index e2695a3..4d02e91 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
@@ -24,6 +24,7 @@ import 
org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilter;
 import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilterBuilder;
 import 
org.apache.eagle.jpm.mr.history.crawler.JobHistorySpoutCollectorInterceptor;
 import org.apache.eagle.jpm.mr.history.publisher.JobStreamPublisher;
+import org.apache.eagle.jpm.mr.history.publisher.StreamPublisher;
 import org.apache.eagle.jpm.mr.history.publisher.StreamPublisherManager;
 import org.apache.eagle.jpm.mr.history.publisher.TaskAttemptStreamPublisher;
 import org.apache.eagle.jpm.mr.history.storm.JobHistorySpout;
@@ -67,8 +68,7 @@ public class MRHistoryJobApplication extends StormApplication 
{
 TopologyBuilder topologyBuilder = new TopologyBuilder();
 String spoutName = "mrHistoryJobSpout";
 int tasks = jhfAppConf.getInt("stormConfig.mrHistoryJobSpoutTasks");
-JobHistorySpoutCollectorInterceptor collectorInterceptor = new 
JobHistorySpoutCollectorInterceptor();
-JobHistorySpout jobHistorySpout = new JobHistorySpout(filter, 
appConfig, collectorInterceptor);
+JobHistorySpout jobHistorySpout = new JobHistorySpout(filter, 
appConfig);
 topologyBuilder.setSpout(
 spoutName,
 jobHistorySpout,
@@ -89,14 +89,11 @@ public class MRHistoryJobApplication extends 
StormApplication {
 String spoutToTaskAttemptSinkName = spoutName + "_to_" + 
taskAttemptSinkBoltName;
 taskAttemptKafkaBoltDeclarer.shuffleGrouping(spoutName, 
spoutToTaskAttemptSinkName);
 
-List streams = new ArrayList<>();
-streams.add(spoutToJobSinkName);
-streams.add(spoutToTaskAttemptSinkName);
-jobHistorySpout.setStreams(streams);
+List streamPublishers = new ArrayList<>();
+streamPublishers.add(new JobStreamPublisher(spoutToJobSinkName));
+streamPublishers.add(new 
TaskAttemptStreamPublisher(spoutToTaskAttemptSinkName));
+jobHistorySpout.setStreamPublishers(streamPublishers);
 
-//4, add stream publisher
-StreamPublisherManager.getInstance().addStreamPublisher(new 
JobStreamPublisher(spoutToJobSinkName, collectorInterceptor));
-StreamPublisherManager.getInstance().addStreamPublisher(new 
TaskAttemptStreamPublisher(spoutToTaskAttemptSinkName, collectorInterceptor));
 return topologyBuilder.createTopology();
 }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e24de5c7/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java
--
diff --git 

incubator-eagle git commit: [MINOR] fix Failed to find entity definition of class TaskAttemptErrorCategor…

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8734c37cb -> 67c915127


[MINOR] fix Failed to find entity definition of class TaskAttemptErrorCategor…

Author: wujinhu 

Closes #743 from wujinhu/EAGLE-795.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/67c91512
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/67c91512
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/67c91512

Branch: refs/heads/master
Commit: 67c91512787ed50ebb976d0218cead62756f0b82
Parents: 8734c37
Author: wujinhu 
Authored: Wed Dec 14 22:43:34 2016 +0800
Committer: wujinhu 
Committed: Wed Dec 14 22:43:34 2016 +0800

--
 .../org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java  | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/67c91512/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
--
diff --git 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
index 2303869..cbbdad3 100644
--- 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
+++ 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java
@@ -37,5 +37,6 @@ public class JPAEntityRepository extends EntityRepository {
 entitySet.add(TaskAttemptCounterAPIEntity.class);
 entitySet.add(JobProcessTimeStampEntity.class);
 entitySet.add(JobCountEntity.class);
+entitySet.add(TaskAttemptErrorCategoryEntity.class);
 }
 }

incubator-eagle git commit: [EAGLE-841] CorrelationSpout reads zk connection from datasource if exists

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 880ba738c -> 7639ff223


[EAGLE-841] CorrelationSpout reads zk connection from datasource if exists

Author: wujinhu 

Closes #741 from wujinhu/EAGLE_841.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7639ff22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7639ff22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7639ff22

Branch: refs/heads/master
Commit: 7639ff2237352884c76e862fd14826cb053bb0fc
Parents: 880ba73
Author: wujinhu 
Authored: Wed Dec 14 17:26:05 2016 +0800
Committer: wujinhu 
Committed: Wed Dec 14 17:26:05 2016 +0800

--
 .../eagle/alert/utils/AlertConstants.java   |  3 ++
 .../alert/engine/spout/CorrelationSpout.java| 38 +++-
 .../engine/topology/CorrelationSpoutTest.java   |  8 +++--
 .../eagle/app/service/ApplicationAction.java| 10 ++
 .../src/main/bin/metadata-ddl.sql   | 10 +++---
 5 files changed, 44 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7639ff22/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
index ee2c28c..2740836 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
@@ -28,4 +28,7 @@ public class AlertConstants {
 public static final String ALERT_SERVICE_ENDPOINT_NAME = "AlertService";
 
 public static final String COORDINATOR = "coordinator";
+
+public static final String KAFKA_BROKER_ZK_BASE_PATH = 
"spout.kafkaBrokerZkBasePath";
+public static final String KAFKA_BROKER_ZK_QUORUM = 
"spout.kafkaBrokerZkQuorum";
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7639ff22/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
index 63e94ca..60a9b98 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
@@ -27,6 +27,7 @@ import backtype.storm.topology.OutputFieldsDeclarer;
 import backtype.storm.topology.base.BaseRichSpout;
 import backtype.storm.tuple.Fields;
 import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
 import org.apache.eagle.alert.coordination.model.SpoutSpec;
@@ -235,8 +236,10 @@ public class CorrelationSpout extends BaseRichSpout 
implements SpoutSpecListener
 
 // build lookup table for scheme
 Map newSchemaName = new HashMap();
+Map> dataSourceProperties = new 
HashMap<>();
 for (Kafka2TupleMetadata ds : 
newMeta.getKafka2TupleMetadataMap().values()) {
 newSchemaName.put(ds.getTopic(), ds.getSchemeCls());
+dataSourceProperties.put(ds.getTopic(), ds.getProperties());
 }
 
 // copy and swap
@@ -248,7 +251,8 @@ public class CorrelationSpout extends BaseRichSpout 
implements SpoutSpecListener
 LOG.warn(MessageFormat.format("try to create new topic {0}, 
but found in the active spout list, this may indicate some inconsistency", 
topic));
 continue;
 }
-KafkaSpoutWrapper newWrapper = createKafkaSpout(conf, context, 
collector, topic, newSchemaName.get(topic), newMeta, sds);
+KafkaSpoutWrapper newWrapper = 
createKafkaSpout(ConfigFactory.parseMap(dataSourceProperties.get(topic)).withFallback(this.config),
+conf, context, collector, topic, 

incubator-eagle git commit: [EAGLE-835] add task failure category

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master d6987af2b -> 9ca2cebae


[EAGLE-835] add task failure category

Author: wujinhu 

Closes #740 from wujinhu/EAGLE-836.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9ca2ceba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9ca2ceba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9ca2ceba

Branch: refs/heads/master
Commit: 9ca2cebaed3dd91a9504326f753cb36a97f150d5
Parents: d6987af
Author: wujinhu 
Authored: Wed Dec 14 14:56:55 2016 +0800
Committer: wujinhu 
Committed: Wed Dec 14 14:56:55 2016 +0800

--
 .../TaskAttemptErrorCategoryEntity.java | 60 
 .../TaskAttemptExecutionAPIEntity.java  | 11 
 .../mr/history/parser/JHFEventReaderBase.java   | 15 -
 .../JobEntityCreationEagleServiceListener.java  |  9 +++
 .../mr/history/parser/TaskFailureListener.java  |  2 +-
 .../org/apache/eagle/jpm/util/Constants.java|  1 +
 6 files changed, 85 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9ca2ceba/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java
--
diff --git 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java
 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java
new file mode 100644
index 000..51254c8
--- /dev/null
+++ 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.eagle.jpm.mr.historyentity;
+
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.meta.*;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Table("eaglejpa_task")
+@ColumnFamily("f")
+@Prefix("taexece")
+@Service(Constants.JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME)
+@TimeSeries(true)
+@Partition({"site"})
+@Indexes({
+@Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false)
+})
+public class TaskAttemptErrorCategoryEntity extends JobBaseAPIEntity {
+@Column("a")
+private long startTime;
+@Column("b")
+private long endTime;
+
+public long getStartTime() {
+return startTime;
+}
+
+public void setStartTime(long startTime) {
+this.startTime = startTime;
+valueChanged("startTime");
+}
+
+public long getEndTime() {
+return endTime;
+}
+
+public void setEndTime(long endTime) {
+this.endTime = endTime;
+valueChanged("endTime");
+}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9ca2ceba/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java
--
diff --git 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java
 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java
index d1210b9..c284341 100644
--- 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java
+++ 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java
@@ -46,8 +46,6 @@ public class TaskAttemptExecutionAPIEntity extends 
JobBaseAPIEntity {
 private String error;
 @Column("f")
 private JobCounters jobCounters;
-@Column("g")
-private String taskAttemptID;
 
 public String getTaskStatus() {
 return taskStatus;
@@ -102,13 

incubator-eagle git commit: [EAGLE-827] coordinator schedule timeout due to last fail

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master b27998f82 -> 743de7330


[EAGLE-827] coordinator schedule timeout due to last fail

Exclusive executor exception could cause take leadership execution exit with 
exception, it will cause follow-up take leadership fails.

Author: Xiancheng Li 

Closes #730 from garrettlish/master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/743de733
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/743de733
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/743de733

Branch: refs/heads/master
Commit: 743de733050e87653f64aca449d5d0a0eb15532d
Parents: b27998f
Author: Xiancheng Li 
Authored: Tue Dec 13 15:50:07 2016 +0800
Committer: wujinhu 
Committed: Tue Dec 13 15:50:07 2016 +0800

--
 .../apache/eagle/alert/coordinator/ExclusiveExecutor.java| 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/743de733/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ExclusiveExecutor.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ExclusiveExecutor.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ExclusiveExecutor.java
index 86a51d9..567e1e2 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ExclusiveExecutor.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ExclusiveExecutor.java
@@ -74,9 +74,13 @@ public class ExclusiveExecutor implements Closeable {
 // do whatever leader work you need to and only exit
 // this method when you want to relinquish leadership
 LOG.info("this is leader node right now..");
-r.run();
-LOG.info("leader node executed done!..");
 executed.set(true);
+try {
+r.run();
+} catch (Throwable t) {
+LOG.warn("failed to run exclusive executor", t);
+}
+LOG.info("leader node executed done!..");
 }
 
 @Override

incubator-eagle git commit: [MINOR] fix health check inject bug

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master e99b2fedf -> 268096f19


[MINOR] fix health check inject bug

Author: wujinhu 

Closes #729 from wujinhu/EAGLE-794.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/268096f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/268096f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/268096f1

Branch: refs/heads/master
Commit: 268096f193926547f76c52cf40cf7775b581dc80
Parents: e99b2fe
Author: wujinhu 
Authored: Mon Dec 12 10:54:30 2016 +0800
Committer: wujinhu 
Committed: Mon Dec 12 10:54:30 2016 +0800

--
 .../eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/268096f1/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
index 7d8dcfd..b90d18b 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
@@ -129,7 +129,7 @@ public class ApplicationHealthCheckServiceImpl extends 
ApplicationHealthCheckSer
 return;
 }
 this.environment.healthChecks().register(appEntity.getAppId(), 
applicationHealthCheck.get());
-currentInjector.injectMembers(applicationHealthCheck);
+currentInjector.injectMembers(applicationHealthCheck.get());
 synchronized (lock) {
 if (!appHealthChecks.containsKey(appEntity.getAppId())) {
 appHealthChecks.put(appEntity.getAppId(), 
applicationHealthCheck);

incubator-eagle git commit: [EAGLE-823] Refactor zkroot and lockpath to solve the problem occured adding zookeeper lock

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 9f4e7633d -> 5da9df822


[EAGLE-823] Refactor zkroot and lockpath to solve the problem occured adding 
zookeeper lock

EAGLE-823 Refactor zkroot and lockpath to solve the problem occured adding 
zookeeper lock

- Refactor zkroot and lockpath to uniform structure.
- Add unit test for eagle-security-hive.

https://issues.apache.org/jira/browse/EAGLE-823

Author: chitin 

Closes #713 from chitin/EAGLE-823.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/5da9df82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/5da9df82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/5da9df82

Branch: refs/heads/master
Commit: 5da9df8221a51252b5e91127d20dea3165553b68
Parents: 9f4e763
Author: chitin 
Authored: Tue Dec 6 17:06:18 2016 +0800
Committer: wujinhu 
Committed: Tue Dec 6 17:06:18 2016 +0800

--
 .../jpm/mr/running/MRRunningJobConfig.java  |  6 +-
 .../jpm/mr/running/parser/MRJobParserTest.java  |  4 +-
 .../spark/running/SparkRunningJobAppConfig.java |  3 +-
 .../java/org/apache/eagle/jpm/util/Utils.java   |  2 +-
 .../org/apache/eagle/jpm/util/UtilsTest.java|  4 +-
 ...HiveJobRunningSourcedStormSpoutProvider.java |  5 +-
 .../hive/jobrunning/TestHiveJobFetchSpout.java  | 93 
 7 files changed, 107 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5da9df82/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobConfig.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobConfig.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobConfig.java
index 119867d..f733b95 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobConfig.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobConfig.java
@@ -32,6 +32,8 @@ public class MRRunningJobConfig implements Serializable {
 
 private static final String ZK_ROOT_PREFIX = "/apps/mr/running";
 
+private static final String JOB_SYMBOL = "/jobs";
+
 public ZKStateConfig getZkStateConfig() {
 return zkStateConfig;
 }
@@ -109,8 +111,8 @@ public class MRRunningJobConfig implements Serializable {
 this.zkStateConfig.zkSessionTimeoutMs = 
config.getInt("zookeeper.zkSessionTimeoutMs");
 this.zkStateConfig.zkRetryTimes = 
config.getInt("zookeeper.zkRetryTimes");
 this.zkStateConfig.zkRetryInterval = 
config.getInt("zookeeper.zkRetryInterval");
-this.zkStateConfig.zkRoot = ZK_ROOT_PREFIX + "/" + 
config.getString("siteId");
-this.zkStateConfig.zkLockPath = 
Utils.makeLockPath(this.zkStateConfig.zkRoot);
+this.zkStateConfig.zkLockPath = Utils.makeLockPath(ZK_ROOT_PREFIX + 
"/" + config.getString("siteId"));
+this.zkStateConfig.zkRoot = ZK_ROOT_PREFIX + "/" + 
config.getString("siteId") + JOB_SYMBOL;
 
 // parse eagle service endpoint
 this.eagleServiceConfig.eagleServiceHost = 
config.getString("service.host");

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5da9df82/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/parser/MRJobParserTest.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/parser/MRJobParserTest.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/parser/MRJobParserTest.java
index 3a71384..f2e581c 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/parser/MRJobParserTest.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/parser/MRJobParserTest.java
@@ -63,8 +63,8 @@ import static 
org.powermock.api.mockito.PowerMockito.mockStatic;
 @PrepareForTest({InputStreamUtils.class, MRJobParser.class, 
URLConnectionUtils.class, Math.class, MRJobEntityCreationHandler.class})
 @PowerMockIgnore({"javax.*", "org.w3c.*", 
"com.sun.org.apache.xerces.*","org.apache.xerces.*"})
 public class MRJobParserTest {
-private static final String ZK_JOB_PATH = 
"/apps/mr/running/sandbox/application_1479206441898_30784/job_1479206441898_30784";
-private static final String ZK_APP_PATH = 
"/apps/mr/running/sandbox/application_1479206441898_30784";
+private static final String ZK_JOB_PATH = 
"/apps/mr/running/sandbox/jobs/application_1479206441898_30784/job_1479206441898_30784";
+

incubator-eagle git commit: [MINOR] Add makeSSS and makeSRS in RouteSpec

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 976edcd86 -> f833e9831


[MINOR] Add makeSSS and makeSRS in RouteSpec

- Add makeSSS and makeSRS in RouteSpec

Author: r7raul1984 

Closes #699 from r7raul1984/ROUTESPEC-MINOR.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/f833e983
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/f833e983
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/f833e983

Branch: refs/heads/master
Commit: f833e9831f473660a084327f67418197b5f00d02
Parents: 976edcd
Author: r7raul1984 
Authored: Tue Dec 6 10:29:05 2016 +0800
Committer: wujinhu 
Committed: Tue Dec 6 10:29:05 2016 +0800

--
 .../alert/coordination/model/RouterSpec.java| 25 
 .../alert/engine/runner/StreamRouterBolt.java   | 15 ++--
 2 files changed, 27 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f833e983/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
index fc13c56..b3877c3 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
@@ -17,9 +17,13 @@
 package org.apache.eagle.alert.coordination.model;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 
 public class RouterSpec {
@@ -66,6 +70,27 @@ public class RouterSpec {
 this.routerSpecs = routerSpecs;
 }
 
+public Map makeSRS() {
+Map newSRS = new HashMap<>();
+this.getRouterSpecs().forEach(t -> {
+if (!newSRS.containsKey(t.getPartition())) {
+newSRS.put(t.getPartition(), new ArrayList<>());
+}
+newSRS.get(t.getPartition()).add(t);
+});
+return newSRS;
+}
+
+public Map makeSSS() {
+Map newSSS = new HashMap<>();
+this.getRouterSpecs().forEach(t -> {
+if (t.getPartition().getSortSpec() != null) {
+newSSS.put(t.getPartition(), t.getPartition().getSortSpec());
+}
+});
+return newSSS;
+}
+
 @Override
 public String toString() {
 return String.format("version:%s-topo:%s, boltSpec:%s", version, 
topologyName, routerSpecs);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f833e983/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
index 29ee771..e37b680 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
@@ -105,12 +105,7 @@ public class StreamRouterBolt extends AbstractStreamBolt 
implements StreamRouter
 sanityCheck(spec);
 
 // figure out added, removed, modified StreamSortSpec
-Map newSSS = new HashMap<>();
-spec.getRouterSpecs().forEach(t -> {
-if (t.getPartition().getSortSpec() != null) {
-newSSS.put(t.getPartition(), t.getPartition().getSortSpec());
-}
-});
+Map newSSS = spec.makeSSS();
 
 Set newStreamIds = newSSS.keySet();
 Set cachedStreamIds = cachedSSS.keySet();
@@ -138,13 +133,7 @@ public class StreamRouterBolt extends AbstractStreamBolt 
implements StreamRouter
 

incubator-eagle git commit: [EAGLE-810] create interface StreamOutputCollector to abstract output…

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 0d1dcc408 -> 976edcd86


[EAGLE-810] create interface StreamOutputCollector to abstract output…

  - create interface StreamOutputCollector to abstract outputcollector

https://issues.apache.org/jira/browse/EAGLE-810

Author: r7raul1984 

Closes #696 from r7raul1984/EAGLE-810.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/976edcd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/976edcd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/976edcd8

Branch: refs/heads/master
Commit: 976edcd868b3b90327efe34ee088db579dfc6e89
Parents: 0d1dcc4
Author: r7raul1984 
Authored: Tue Dec 6 10:13:22 2016 +0800
Committer: wujinhu 
Committed: Tue Dec 6 10:13:22 2016 +0800

--
 ...ertBoltOutputCollectorThreadSafeWrapper.java |  6 +-
 .../impl/AlertBoltOutputCollectorWrapper.java   |  6 +-
 .../engine/router/StreamOutputCollector.java| 33 ++
 .../router/impl/StormOutputCollector.java   | 65 
 .../impl/StreamRouterBoltOutputCollector.java   | 41 +---
 .../eagle/alert/engine/runner/AlertBolt.java|  5 +-
 .../alert/engine/runner/StreamRouterBolt.java   |  3 +-
 ...oltOutputCollectorThreadSafeWrapperTest.java |  3 +-
 .../TestStreamRouterBoltOutputCollector.java| 31 ++
 9 files changed, 144 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/976edcd8/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
index 33d502e..185853d 100755
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
@@ -18,7 +18,7 @@ package org.apache.eagle.alert.engine.evaluator.impl;
 
 import org.apache.eagle.alert.engine.AlertStreamCollector;
 import org.apache.eagle.alert.engine.model.AlertStreamEvent;
-import backtype.storm.task.OutputCollector;
+import org.apache.eagle.alert.engine.router.StreamOutputCollector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,14 +40,14 @@ import java.util.concurrent.atomic.AtomicLong;
  * 
  */
 public class AlertBoltOutputCollectorThreadSafeWrapper implements 
AlertStreamCollector {
-private final OutputCollector delegate;
+private final StreamOutputCollector delegate;
 private final LinkedBlockingQueue queue;
 private static final Logger LOG = 
LoggerFactory.getLogger(AlertBoltOutputCollectorThreadSafeWrapper.class);
 private final AtomicLong lastFlushTime = new 
AtomicLong(System.currentTimeMillis());
 private final AutoAlertFlusher flusher;
 private static final int MAX_ALERT_DELAY_SECS = 10;
 
-public AlertBoltOutputCollectorThreadSafeWrapper(OutputCollector 
outputCollector) {
+public AlertBoltOutputCollectorThreadSafeWrapper(StreamOutputCollector 
outputCollector) {
 this.delegate = outputCollector;
 this.queue = new LinkedBlockingQueue<>();
 this.flusher = new AutoAlertFlusher(this);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/976edcd8/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
index af2b9f8..3053e6e 100755
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
@@ -25,22 +25,22 @@ import 

incubator-eagle git commit: [MINOR] support group by in siddhi pattern match

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 3a84a2c62 -> e5e215e0b


[MINOR] support group by in siddhi pattern match

Author: wujinhu 

Closes #708 from wujinhu/EAGLE-792.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e5e215e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e5e215e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e5e215e0

Branch: refs/heads/master
Commit: e5e215e0bc0d4829e120069cab92922e0fef5f5a
Parents: 3a84a2c
Author: wujinhu 
Authored: Mon Dec 5 12:24:55 2016 +0800
Committer: wujinhu 
Committed: Mon Dec 5 12:24:55 2016 +0800

--
 .../interpreter/PolicyExecutionPlannerImpl.java | 35 +---
 .../interpreter/PolicyInterpreterTest.java  | 34 +++
 2 files changed, 64 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e5e215e0/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlannerImpl.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlannerImpl.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlannerImpl.java
index 82bb64f..1f46298 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlannerImpl.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlannerImpl.java
@@ -32,10 +32,8 @@ import org.wso2.siddhi.query.api.execution.ExecutionElement;
 import org.wso2.siddhi.query.api.execution.query.Query;
 import org.wso2.siddhi.query.api.execution.query.input.handler.StreamHandler;
 import org.wso2.siddhi.query.api.execution.query.input.handler.Window;
-import org.wso2.siddhi.query.api.execution.query.input.stream.InputStream;
-import org.wso2.siddhi.query.api.execution.query.input.stream.JoinInputStream;
-import 
org.wso2.siddhi.query.api.execution.query.input.stream.SingleInputStream;
-import org.wso2.siddhi.query.api.execution.query.input.stream.StateInputStream;
+import org.wso2.siddhi.query.api.execution.query.input.state.*;
+import org.wso2.siddhi.query.api.execution.query.input.stream.*;
 import org.wso2.siddhi.query.api.execution.query.output.stream.OutputStream;
 import org.wso2.siddhi.query.api.execution.query.selection.OutputAttribute;
 import org.wso2.siddhi.query.api.execution.query.selection.Selector;
@@ -61,6 +59,7 @@ class PolicyExecutionPlannerImpl implements 
PolicyExecutionPlanner {
 
 private final String executionPlan;
 private final Map effectiveInputStreams;
+private final Map effectiveInputStreamsAlias;
 private final Map effectiveOutputStreams;
 private final Map effectivePartitions;
 private final PolicyExecutionPlan policyExecutionPlan;
@@ -68,6 +67,7 @@ class PolicyExecutionPlannerImpl implements 
PolicyExecutionPlanner {
 public PolicyExecutionPlannerImpl(String executionPlan) throws Exception {
 this.executionPlan = executionPlan;
 this.effectiveInputStreams = new HashMap<>();
+this.effectiveInputStreamsAlias = new HashMap<>();
 this.effectiveOutputStreams = new HashMap<>();
 this.effectivePartitions = new HashMap<>();
 this.policyExecutionPlan = doParse();
@@ -173,6 +173,9 @@ class PolicyExecutionPlannerImpl implements 
PolicyExecutionPlanner {
 for (String streamId : 
inputStream.getUniqueStreamIds()) {
 streamGroupBy.put(streamId, new 
ArrayList<>());
 }
+
+
collectStreamReferenceIdMapping(((StateInputStream)inputStream).getStateElement());
+
 for (Variable variable : groupBy) {
 // Not stream not set, then should be all 
streams' same field
 if (variable.getStreamId() == null) {
@@ -180,7 +183,12 @@ class PolicyExecutionPlannerImpl implements 
PolicyExecutionPlanner {
 
streamGroupBy.get(streamId).add(variable);
 }
 } else {
-String streamId = 
retrieveStreamId(variable, 

incubator-eagle git commit: [EAGLE-805] sync some operation in RunningJobManager

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master e6c4b0a73 -> aef7ea36c


[EAGLE-805] sync some operation in RunningJobManager

EAGLE-805 sync some operation in RunningJobManager
- Acquire lock when recoverYarnApp and recover methods were invoked.
- Add unit test for the two methods under high concurrency.
- Modify lockPath to avoid exception when curator.getChildren().

https://issues.apache.org/jira/browse/EAGLE-805

Author: chitin 

Closes #700 from chitin/EAGLE-805.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/aef7ea36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/aef7ea36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/aef7ea36

Branch: refs/heads/master
Commit: aef7ea36c1473b29f76420dd67d646e738f7d10b
Parents: e6c4b0a
Author: chitin 
Authored: Fri Dec 2 16:00:23 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 2 16:00:23 2016 +0800

--
 .../jpm/mr/running/MRRunningJobManagerTest.java | 112 +--
 .../src/test/resources/jobInfo_805.json |   4 +
 .../java/org/apache/eagle/jpm/util/Utils.java   |   2 +-
 .../jpm/util/jobrecover/RunningJobManager.java  | 105 ++---
 .../org/apache/eagle/jpm/util/UtilsTest.java|   2 +-
 5 files changed, 170 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/aef7ea36/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/MRRunningJobManagerTest.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/MRRunningJobManagerTest.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/MRRunningJobManagerTest.java
index 55f76e2..4c52e10 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/MRRunningJobManagerTest.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/test/java/org/apache/eagle/jpm/mr/running/MRRunningJobManagerTest.java
@@ -24,18 +24,21 @@ import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.curator.test.TestingServer;
 import org.apache.curator.utils.CloseableUtils;
 import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager;
+import org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity;
 import org.apache.eagle.jpm.util.jobrecover.RunningJobManager;
 import org.apache.zookeeper.CreateMode;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
 import org.junit.runner.RunWith;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 import org.slf4j.LoggerFactory;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -60,16 +63,14 @@ public class MRRunningJobManagerTest {
 private static MRRunningJobConfig.EndpointConfig endpointConfig;
 private static MRRunningJobConfig.ZKStateConfig zkStateConfig;
 private static org.slf4j.Logger log = mock(org.slf4j.Logger.class);
+private static final int BUFFER_SIZE = 4096;
+private static final String LOCKS_BASE_PATH = "/locks";
 
 @BeforeClass
 public static void setupZookeeper() throws Exception {
 zk = new TestingServer();
 curator = CuratorFrameworkFactory.newClient(zk.getConnectString(), new 
ExponentialBackoffRetry(1000, 3));
 curator.start();
-curator.create()
-.creatingParentsIfNeeded()
-.withMode(CreateMode.PERSISTENT)
-.forPath(SHARE_RESOURCES);
 MRRunningJobConfig mrRunningJobConfig = 
MRRunningJobConfig.newInstance(config);
 zkStateConfig = mrRunningJobConfig.getZkStateConfig();
 zkStateConfig.zkQuorum = zk.getConnectString();
@@ -80,14 +81,33 @@ public class MRRunningJobManagerTest {
 
 @AfterClass
 public static void teardownZookeeper() throws Exception {
+CloseableUtils.closeQuietly(curator);
+CloseableUtils.closeQuietly(zk);
+}
+
+@Before
+public void createPath() throws Exception {
+if(curator.checkExists().forPath(SHARE_RESOURCES) == null) {
+curator.create()
+.creatingParentsIfNeeded()
+.withMode(CreateMode.PERSISTENT)
+.forPath(SHARE_RESOURCES);
+}
+}
+
+@After
+public void 

incubator-eagle git commit: [MINOR] Sleep 1 ms after versionId was generated

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master a292c1556 -> e6c4b0a73


[MINOR] Sleep 1 ms after versionId was generated

[MINOR] Sleep 1 ms after versionId was generated
- VersionId is the same with one generated in last loop randomly, the test 
failed in this case.

Author: chitin 

Closes #706 from chitin/MINOR_JDBCTEST.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e6c4b0a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e6c4b0a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e6c4b0a7

Branch: refs/heads/master
Commit: e6c4b0a731942c9c4bf746ecc6b067429d755ff6
Parents: a292c15
Author: chitin 
Authored: Fri Dec 2 15:58:15 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 2 15:58:15 2016 +0800

--
 .../java/org/apache/eagle/alert/metadata/impl/JdbcImplTest.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e6c4b0a7/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/alert/metadata/impl/JdbcImplTest.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/alert/metadata/impl/JdbcImplTest.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/alert/metadata/impl/JdbcImplTest.java
index 9d188c4..dc7c657 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/alert/metadata/impl/JdbcImplTest.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/alert/metadata/impl/JdbcImplTest.java
@@ -177,7 +177,7 @@ public class JdbcImplTest {
 List reservedOnes = new ArrayList<>();
 for (int i = 0; i < 10; i++) {
 ScheduleState state = new ScheduleState();
-String versionId = "state-" + System.currentTimeMillis();
+String versionId = "state-" + (System.currentTimeMillis() + i);
 state.setVersion(versionId);
 state.setGenerateTime(String.valueOf(new Date().getTime()));
 dao.addScheduleState(state);
@@ -190,6 +190,8 @@ public class JdbcImplTest {
 Assert.assertTrue(scheduleStates.size() == maxCapacity);
 List TargetVersions = new ArrayList<>();
 scheduleStates.stream().forEach(state -> 
TargetVersions.add(state.getVersion()));
+LOG.debug(reservedOnes.toString());
+LOG.debug(TargetVersions.toString());
 Assert.assertTrue(CollectionUtils.isEqualCollection(reservedOnes, 
TargetVersions));
 }
 }

incubator-eagle git commit: [MINOR] add missing licences

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master a0f134422 -> a292c1556


[MINOR] add missing licences

Author: Xiancheng Li 

Closes #704 from garrettlish/master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a292c155
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a292c155
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a292c155

Branch: refs/heads/master
Commit: a292c1556058126e5fcb897cab58a31ab6dbb49d
Parents: a0f1344
Author: Xiancheng Li 
Authored: Fri Dec 2 11:02:29 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 2 11:02:29 2016 +0800

--
 .../eagle/alert/engine/StormMultiCountMetric.java   | 16 
 .../apache/eagle/alert/engine/StreamCounter.java| 16 
 .../service/metric/TestEagleMetricResource.java | 16 
 3 files changed, 48 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a292c155/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StormMultiCountMetric.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StormMultiCountMetric.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StormMultiCountMetric.java
index aa97b57..26760ab 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StormMultiCountMetric.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StormMultiCountMetric.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.eagle.alert.engine;
 
 import backtype.storm.metric.api.MultiCountMetric;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a292c155/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StreamCounter.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StreamCounter.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StreamCounter.java
index ff96c30..2d9ea69 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StreamCounter.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/StreamCounter.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.eagle.alert.engine;
 
 public interface StreamCounter {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a292c155/eagle-core/eagle-query/eagle-service-base/src/test/java/org/apache/eagle/service/metric/TestEagleMetricResource.java
--
diff --git 

incubator-eagle git commit: [EAGLE-817] add KillOptions when stop a topology

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master df9d9b085 -> c52e23c74


[EAGLE-817] add KillOptions when stop a topology

Author: wujinhu 

Closes #702 from wujinhu/EAGLE-817.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c52e23c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c52e23c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c52e23c7

Branch: refs/heads/master
Commit: c52e23c748ba5f0f497622f8a2d50bcd7de340a9
Parents: df9d9b0
Author: wujinhu 
Authored: Thu Dec 1 15:48:49 2016 +0800
Committer: wujinhu 
Committed: Thu Dec 1 15:48:49 2016 +0800

--
 .../app/environment/impl/StormExecutionRuntime.java | 12 
 1 file changed, 8 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c52e23c7/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
index 4a9b2b7..a4e9fac 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
@@ -43,6 +43,7 @@ public class StormExecutionRuntime implements 
ExecutionRuntime

incubator-eagle git commit: [MINOR] remove system property for metadata.metadataDAO in JDBCMetadataStore

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 68a3e8bde -> df9d9b085


[MINOR] remove system property for metadata.metadataDAO in JDBCMetadataStore

Author: wujinhu 

Closes #703 from wujinhu/EAGLE-791.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/df9d9b08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/df9d9b08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/df9d9b08

Branch: refs/heads/master
Commit: df9d9b085714c55c1bdc5a4d4e71a96c5d0c9c30
Parents: 68a3e8b
Author: wujinhu 
Authored: Thu Dec 1 15:40:00 2016 +0800
Committer: wujinhu 
Committed: Thu Dec 1 15:40:00 2016 +0800

--
 .../org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java | 1 -
 eagle-server/src/main/resources/application.conf| 1 -
 2 files changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/df9d9b08/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java
--
diff --git 
a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java
 
b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java
index 9a7e7ab..2e12e23 100644
--- 
a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java
+++ 
b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java
@@ -34,7 +34,6 @@ import javax.sql.DataSource;
 public class JDBCMetadataStore extends MetadataStore {
 @Override
 protected void configure() {
-System.setProperty(MetadataUtils.META_DATA + "." + 
MetadataUtils.ALERT_META_DATA_DAO, 
JdbcMetadataDaoImpl.class.getCanonicalName());
 
bind(IMetadataDao.class).to(JdbcMetadataDaoImpl.class).in(Singleton.class);
 
bind(DataSource.class).toProvider(JDBCDataSourceProvider.class).in(Singleton.class);
 
bind(JDBCDataSourceConfig.class).toProvider(JDBCMetadataStoreConfigProvider.class).in(Singleton.class);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/df9d9b08/eagle-server/src/main/resources/application.conf
--
diff --git a/eagle-server/src/main/resources/application.conf 
b/eagle-server/src/main/resources/application.conf
index 20f5b2e..f30c54f 100644
--- a/eagle-server/src/main/resources/application.conf
+++ b/eagle-server/src/main/resources/application.conf
@@ -71,7 +71,6 @@ storage {
 # -
 metadata {
   store = org.apache.eagle.metadata.service.memory.MemoryMetadataStore
-  metadataDao = org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl
   jdbc {
 username = "root"
 password = ""

incubator-eagle git commit: [EAGLE-801] Add unit test for eagle-core module

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 4d4d8c0ea -> 7976858ee


[EAGLE-801] Add unit test for eagle-core module

EAGLE-801 Add unit test for eagle-core module
- Add unit test for eagle-core module.
https://issues.apache.org/jira/browse/EAGLE-801

Author: chitin 
Author: Lingang Deng 

Closes #685 from chitin/EAGLE-801.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7976858e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7976858e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7976858e

Branch: refs/heads/master
Commit: 7976858ee068ca4a85d67f70f7bf574f3e75115f
Parents: 4d4d8c0
Author: chitin 
Authored: Mon Nov 28 16:46:03 2016 +0800
Committer: wujinhu 
Committed: Mon Nov 28 16:46:03 2016 +0800

--
 eagle-core/eagle-app/eagle-app-base/pom.xml |  12 +
 .../app/service/ApplicationActionTest.java  |  58 +
 ...estApplicationHealthCheckEmailPublisher.java |  64 +
 .../TestApplicationHealthCheckServiceImpl.java  | 101 
 .../TestApplicationManagementServiceImpl.java   | 181 +++
 .../org/apache/eagle/common/TestBase64.java |  32 +++
 .../eagle/common/TestEagleExceptionWrapper.java |  32 +++
 .../eagle/common/TestSerializableUtils.java |   1 -
 .../eagle/common/config/TestConfigParser.java   |  36 +++
 .../eagle-metadata/eagle-metadata-base/pom.xml  |   4 +
 .../eagle/metadata/model/TestConfiguration.java |  67 ++
 .../metadata/persistence/MetadataStoreTest.java |  28 +++
 .../TestMetadataStoreModuleFactory.java |  33 +++
 .../TestApplicationEntityServiceMemoryImpl.java | 231 +++
 .../TestSiteEntityServiceMemoryImpl.java|  97 
 .../src/test/resources/application-test.conf|  31 +++
 .../src/test/resources/application-test.xml |  17 ++
 .../ApplicationEntityServiceJDBCImplTest.java   |  66 ++
 .../src/test/resources/application.conf |   6 +-
 19 files changed, 1095 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7976858e/eagle-core/eagle-app/eagle-app-base/pom.xml
--
diff --git a/eagle-core/eagle-app/eagle-app-base/pom.xml 
b/eagle-core/eagle-app/eagle-app-base/pom.xml
index 5e55239..a5e295e 100644
--- a/eagle-core/eagle-app/eagle-app-base/pom.xml
+++ b/eagle-core/eagle-app/eagle-app-base/pom.xml
@@ -113,5 +113,17 @@
 
 
 
+
+org.powermock
+powermock-module-junit4
+${powermock.version}
+test
+
+
+org.powermock
+powermock-api-mockito
+${powermock.version}
+test
+
 
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7976858e/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/service/ApplicationActionTest.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/service/ApplicationActionTest.java
 
b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/service/ApplicationActionTest.java
index d076bf7..76bb76a 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/service/ApplicationActionTest.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/service/ApplicationActionTest.java
@@ -18,10 +18,24 @@ package org.apache.eagle.app.service;
 
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
+import org.apache.commons.collections.map.HashedMap;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.metadata.IMetadataDao;
+import org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl;
+import org.apache.eagle.alert.metric.MetricConfigs;
+import org.apache.eagle.app.Application;
+import org.apache.eagle.app.environment.impl.StaticEnvironment;
+import org.apache.eagle.metadata.model.ApplicationDesc;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.model.SiteEntity;
 import org.junit.Assert;
 import org.junit.Test;
+import static org.mockito.Mockito.*;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public class ApplicationActionTest {
 /**
@@ -45,4 +59,48 @@ public class ApplicationActionTest {
 Assert.assertEquals("appConfig.withFallback(envConfig): appConfig will 
override envConfig, envConfig is used as default config",
 

incubator-eagle git commit: [MINOR] change delay strategy of topology check app

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master de061d599 -> 52dc7a5d9


[MINOR] change delay strategy of topology check app

Author: wujinhu 

Closes #682 from wujinhu/EAGLE-789.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/52dc7a5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/52dc7a5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/52dc7a5d

Branch: refs/heads/master
Commit: 52dc7a5d99ed37b7778813c8b6483a4227e7c0ee
Parents: de061d5
Author: wujinhu 
Authored: Thu Nov 24 22:19:56 2016 +0800
Committer: wujinhu 
Committed: Thu Nov 24 22:19:56 2016 +0800

--
 ...adoopQueueRunningApplicationHealthCheck.java |  2 +-
 .../SparkHistoryJobApplicationHealthCheck.java  |  2 +-
 .../TopologyCheckApplicationHealthCheck.java| 36 +---
 3 files changed, 25 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52dc7a5d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
--
diff --git 
a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
 
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
index 61c7687..6a7535c 100644
--- 
a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
+++ 
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
@@ -71,7 +71,7 @@ public class HadoopQueueRunningApplicationHealthCheck extends 
ApplicationHealthC
 
.metricName(HadoopClusterConstants.MetricName.HADOOP_CLUSTER_ALLOCATED_MEMORY)
 .startTime(System.currentTimeMillis() - 24 * 60 * 6L)
 .endTime(System.currentTimeMillis())
-.pageSize(Integer.MAX_VALUE)
+.pageSize(10)
 .send();
 List> results = response.getObj();
 long currentProcessTimeStamp = 
results.get(0).get("value").get(0).longValue();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52dc7a5d/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
--
diff --git 
a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
 
b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
index fe6bd5a..d6f5031 100644
--- 
a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
+++ 
b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
@@ -65,7 +65,7 @@ public class SparkHistoryJobApplicationHealthCheck extends 
ApplicationHealthChec
 
 GenericServiceAPIResponseEntity response = client
 .search(query)
-.startTime(System.currentTimeMillis() - 24 * 60 * 6L)
+.startTime(System.currentTimeMillis() - 12 * 60 * 6L)
 .endTime(System.currentTimeMillis())
 .pageSize(10)
 .send();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52dc7a5d/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
--
diff --git 
a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
 
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
index 5115af6..7860cb5 100644
--- 
a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
+++ 
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
@@ -59,19 +59,11 @@ public class TopologyCheckApplicationHealthCheck extends 
ApplicationHealthCheckB
 message += String.format("Application is not RUNNING, status 
is %s. ", status.toString());
 }
 
-String query = 

incubator-eagle git commit: [MINOR] optimize health check: add some information

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master adeaa0f9e -> 3becca689


[MINOR] optimize health check: add some information

Author: wujinhu 

Closes #679 from wujinhu/healthFix.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3becca68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3becca68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3becca68

Branch: refs/heads/master
Commit: 3becca68929cd005416605da0b9be4cbdb0c0cba
Parents: adeaa0f
Author: wujinhu 
Authored: Thu Nov 24 12:35:05 2016 +0800
Committer: wujinhu 
Committed: Thu Nov 24 12:35:05 2016 +0800

--
 .../ApplicationHealthCheckEmailPublisher.java   | 29 ++--
 .../impl/ApplicationHealthCheckServiceImpl.java |  4 +--
 .../src/main/resources/application.conf |  1 -
 3 files changed, 29 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3becca68/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
index c7d7e0f..8374b38 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
@@ -26,6 +26,7 @@ import org.apache.velocity.VelocityContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.InetAddress;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -72,8 +73,15 @@ public class ApplicationHealthCheckEmailPublisher implements 
ApplicationHealthCh
 
 final VelocityContext context = new VelocityContext();
 Map appMsgs = new HashMap<>();
+int unhealthyCount = 0;
+int healthyCount = 0;
 for (String appId : results.keySet()) {
 appMsgs.put(appId, results.get(appId).getMessage());
+if (!results.get(appId).isHealthy()) {
+unhealthyCount++;
+} else {
+healthyCount++;
+}
 }
 Map unHealthyContext = new HashMap<>();
 unHealthyContext.put("appMsgs", appMsgs);
@@ -81,11 +89,28 @@ public class ApplicationHealthCheckEmailPublisher 
implements ApplicationHealthCh
 unHealthyContext.put("healthCheckUrl", "http://; + 
config.getString(SERVICE_HOST) + ":" + HEALTH_CHECK_PORT + "/healthcheck");
 context.put(UNHEALTHY_CONTEXT, unHealthyContext);
 
+String subject = "";
+if (healthyCount > 0) {
+subject += healthyCount + " healthy app(s)";
+}
+if (unhealthyCount > 0) {
+if (!subject.isEmpty()) {
+subject += ", ";
+}
+subject += unhealthyCount + " unhealthy app(s)";
+}
+subject = config.getString(CONF_MAIL_SUBJECT) + ": " + subject;
 EagleMailClient client = new EagleMailClient(properties);
-success = client.send(config.getString(CONF_MAIL_SENDER),
+String hostname = InetAddress.getLocalHost().getHostName();
+if (!hostname.endsWith(".com")) {
+//avoid invalid host exception
+hostname += ".com";
+}
+success = client.send(
+System.getProperty("user.name") + "@" + hostname,
 recipients,
 config.hasPath(CONF_MAIL_CC) ? 
config.getString(CONF_MAIL_CC) : null,
-config.getString(CONF_MAIL_SUBJECT),
+subject,
 config.getString(CONF_MAIL_TEMPLATE),
 context,
 null);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3becca68/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
--
diff --git 

incubator-eagle git commit: [MINOR] health check optimize

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8d7f81e1c -> 929a21da7


[MINOR] health check optimize

Author: wujinhu 

Closes #677 from wujinhu/healthCheckOptimize.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/929a21da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/929a21da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/929a21da

Branch: refs/heads/master
Commit: 929a21da7182986dc62db953639e27cc983f6f23
Parents: 8d7f81e
Author: wujinhu 
Authored: Wed Nov 23 20:09:32 2016 +0800
Committer: wujinhu 
Committed: Wed Nov 23 20:09:32 2016 +0800

--
 .../ApplicationHealthCheckPublisher.java|  4 +-
 .../impl/ApplicationHealthCheckBase.java| 12 
 .../ApplicationHealthCheckEmailPublisher.java   | 20 --
 .../impl/ApplicationHealthCheckServiceImpl.java | 68 +++-
 .../eagle/app/spi/ApplicationProvider.java  |  1 -
 .../src/main/resources/HealthCheckTemplate.vm   | 18 +-
 ...adoopQueueRunningApplicationHealthCheck.java | 13 ++--
 .../MRHistoryJobApplicationHealthCheck.java | 10 +--
 .../JobEntityCreationEagleServiceListener.java  |  5 +-
 .../SparkHistoryJobApplicationHealthCheck.java  | 12 ++--
 .../src/main/resources/application.conf |  3 +-
 .../TopologyCheckApplicationHealthCheck.java| 10 +--
 12 files changed, 127 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
index 9469521..4a8d2e7 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationHealthCheckPublisher.java
@@ -19,6 +19,8 @@ package org.apache.eagle.app.service;
 
 import com.codahale.metrics.health.HealthCheck;
 
+import java.util.Map;
+
 public interface ApplicationHealthCheckPublisher {
-void onUnHealthApplication(String appId, HealthCheck.Result result);
+void onUnHealthApplication(Map results);
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
index 104e76d..75b7c2d 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
@@ -23,6 +23,9 @@ import com.typesafe.config.Config;
 import org.apache.eagle.metadata.model.ApplicationEntity;
 import org.apache.eagle.metadata.service.ApplicationEntityService;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
 public abstract class ApplicationHealthCheckBase extends HealthCheck {
 private static final String APP_ID_PATH = "appId";
 protected static final long DEFAULT_MAX_DELAY_TIME = 2 * 60 * 60 * 1000L;
@@ -41,4 +44,13 @@ public abstract class ApplicationHealthCheckBase extends 
HealthCheck {
 ApplicationEntity applicationEntity = 
applicationEntityService.getByUUIDOrAppId(null, config.getString(APP_ID_PATH));
 return applicationEntity.getStatus();
 }
+
+protected String printMessages(String ... messages) {
+StringWriter sw = new StringWriter();
+PrintWriter pw = new PrintWriter(sw, true);
+for (int i = 0; i < messages.length; i++) {
+pw.println(messages[i]);
+}
+return sw.getBuffer().toString();
+}
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/929a21da/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckEmailPublisher.java
--
diff --git 

incubator-eagle git commit: [EAGLE-776] add unit test for eagle-alert-parent

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 7499be694 -> 8d7f81e1c


[EAGLE-776] add unit test for eagle-alert-parent

add unit test for eagle-alert-parent

https://issues.apache.org/jira/browse/EAGLE-776

Author: koone 

Closes #673 from koone/EAGLE-777.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8d7f81e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8d7f81e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8d7f81e1

Branch: refs/heads/master
Commit: 8d7f81e1cdcc61e16769677dda2c371897f25dca
Parents: 7499be6
Author: koone 
Authored: Wed Nov 23 14:29:48 2016 +0800
Committer: wujinhu 
Committed: Wed Nov 23 14:29:48 2016 +0800

--
 .../alert-metadata-service/pom.xml  |  13 +-
 .../impl/TopologyMgmtResourceImplTest.java  |  72 
 .../src/test/resources/application.conf |  10 +-
 .../alert-metadata/pom.xml  |  12 +
 .../eagle/alert/metadata/MetadataUtils.java |   1 +
 .../metadata/impl/InMemMetadataDaoImpl.java |   6 +-
 .../eagle/alert/metadata/TestMetadataUtils.java |  59 
 .../eagle/alert/metadata/impl/InMemoryTest.java | 117 +++
 .../eagle/alert/metadata/impl/JdbcImplTest.java | 164 +
 .../alert/metadata/impl/MongoImplTest.java  | 344 ++
 .../alert/resource/impl/InMemoryTest.java   |  48 ---
 .../alert/resource/impl/JdbcImplTest.java   | 165 -
 .../alert/resource/impl/MongoImplTest.java  | 345 ---
 13 files changed, 791 insertions(+), 565 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8d7f81e1/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/pom.xml
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/pom.xml
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/pom.xml
index 9d5e8f1..cf1f0fc 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/pom.xml
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/pom.xml
@@ -84,7 +84,18 @@
 io.swagger
 swagger-jaxrs
 
-
+
+org.powermock
+powermock-module-junit4
+${powermock.version}
+test
+
+
+org.powermock
+powermock-api-mockito
+${powermock.version}
+test
+
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8d7f81e1/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
index e46213e..b9a7634 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
@@ -18,13 +18,40 @@
 
 package org.apache.eagle.service.topology.resource.impl;
 
+import backtype.storm.StormSubmitter;
+import backtype.storm.generated.ClusterSummary;
+import backtype.storm.generated.Nimbus;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.generated.TopologySummary;
+import backtype.storm.utils.NimbusClient;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.IMetadataDao;
+import org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl;
+import org.apache.eagle.alert.metadata.impl.MetadataDaoFactory;
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import 

incubator-eagle git commit: [EAGLE-786] Add unit test for eagle-jpm-mr-running's MRRunningJobFetc…

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master a39ca45cb -> 7499be694


[EAGLE-786] Add unit test for eagle-jpm-mr-running's MRRunningJobFetc…

 - Add unit test for eagle-jpm-mr-running's MRRunningJobFetchSpout

https://issues.apache.org/jira/browse/EAGLE-786

Author: r7raul1984 

Closes #668 from r7raul1984/EAGLE-786.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7499be69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7499be69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7499be69

Branch: refs/heads/master
Commit: 7499be694a6ae2c334751d49927cc5eaac6f5f2b
Parents: a39ca45
Author: r7raul1984 
Authored: Wed Nov 23 11:40:19 2016 +0800
Committer: wujinhu 
Committed: Wed Nov 23 11:40:19 2016 +0800

--
 eagle-jpm/eagle-jpm-mr-running/pom.xml  |  12 ++
 .../running/storm/MRRunningJobFetchSpout.java   |  64 
 .../mr/running/MRRunningJobApplicationTest.java | 147 ++-
 .../application_1479206441898_35341.json|  35 +
 .../test/resources/previousmrrunningapp.json|  64 
 .../test/resources/thistimemrrunningapp.json|  35 +
 6 files changed, 326 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7499be69/eagle-jpm/eagle-jpm-mr-running/pom.xml
--
diff --git a/eagle-jpm/eagle-jpm-mr-running/pom.xml 
b/eagle-jpm/eagle-jpm-mr-running/pom.xml
index bb0ac5a..b4db22b 100644
--- a/eagle-jpm/eagle-jpm-mr-running/pom.xml
+++ b/eagle-jpm/eagle-jpm-mr-running/pom.xml
@@ -69,6 +69,18 @@
 eagle-app-base
 ${project.version}
 
+
+org.powermock
+powermock-module-junit4
+${powermock.version}
+test
+
+
+org.powermock
+powermock-api-mockito
+${powermock.version}
+test
+
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7499be69/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobFetchSpout.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobFetchSpout.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobFetchSpout.java
index 7c910e7..cc5df84 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobFetchSpout.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobFetchSpout.java
@@ -18,6 +18,7 @@
 
 package org.apache.eagle.jpm.mr.running.storm;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.eagle.jpm.mr.running.MRRunningJobConfig;
 import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager;
 import org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity;
@@ -87,40 +88,31 @@ public class MRRunningJobFetchSpout extends BaseRichSpout {
 } else {
 apps = 
resourceFetcher.getResource(Constants.ResourceType.RUNNING_MR_JOB);
 LOG.info("get {} apps from resource manager", apps.size());
-Set running = new HashSet<>();
-for (AppInfo appInfo : apps) {
-running.add(appInfo.getId());
-}
-Iterator appIdIterator = 
this.runningYarnApps.iterator();
-while (appIdIterator.hasNext()) {
-String appId = appIdIterator.next();
-boolean hasFinished = true;
-for (AppInfo appInfo : apps) {
-if (appId.equals(appInfo.getId())) {
-hasFinished = false;
-}
-}
 
-if (hasFinished) {
-try {
-Map result = 
this.runningJobManager.recoverYarnApp(appId);
-if (result.size() > 0) {
-if (mrApps == null) {
-mrApps = new HashMap<>();
-}
-mrApps.put(appId, result);
-AppInfo appInfo = 
result.get(result.keySet().iterator().next()).getAppInfo();
-
appInfo.setState(Constants.AppState.FINISHED.toString());
-apps.add(appInfo);
+Set 

incubator-eagle git commit: [EAGLE-783] Add unit test for eagle-jpm-util.

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 4874433a5 -> d6e78bec2


[EAGLE-783] Add unit test for eagle-jpm-util.

 - Add unit test for eagle-jpm-util.

https://issues.apache.org/jira/browse/EAGLE-783

Author: r7raul1984 

Closes #663 from r7raul1984/EAGLE-783.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/d6e78bec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/d6e78bec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/d6e78bec

Branch: refs/heads/master
Commit: d6e78bec28b804f3c6bfaec2f69774a717f5cbaa
Parents: 4874433
Author: r7raul1984 
Authored: Thu Nov 17 20:28:09 2016 +0800
Committer: wujinhu 
Committed: Thu Nov 17 20:28:09 2016 +0800

--
 eagle-jpm/eagle-jpm-util/pom.xml|  12 +++
 .../jpm/util/resourcefetch/model/AppInfo.java   |  25 +
 .../util/resourcefetch/model/ClusterInfo.java   |  18 +++-
 .../jpm/util/DefaultJobIdPartitionerTest.java   |  29 +
 .../apache/eagle/jpm/util/JSONUtilsTest.java|  79 ++
 .../jpm/util/JobNameNormalizationTest.java  |  45 
 .../org/apache/eagle/jpm/util/UtilsTest.java|  67 
 .../jobcounter/JobCountersSerDeserTest.java |  46 
 .../resourcefetch/RMResourceFetcherTest.java| 105 +++
 .../resourcefetch/ha/HAURLSelectorImplTest.java | 102 ++
 .../url/JobListServiceURLBuilderImplTest.java   |  37 +++
 ...arkCompleteJobServiceURLBuilderImplTest.java |  30 ++
 .../url/SparkJobServiceURLBuilderImplTest.java  |  30 ++
 .../jpm/util/resourcefetch/url/URLUtilTest.java |  35 +++
 .../src/test/resources/clusterinfo.json |  15 +++
 .../src/test/resources/mrcompleteapp.json   |  54 ++
 .../src/test/resources/mrrunningapp.json|  64 +++
 .../src/test/resources/sparkcompleteapp.json|  64 +++
 .../src/test/resources/sparkrunningapp.json |  64 +++
 eagle-security/eagle-security-oozie-web/pom.xml |   4 -
 pom.xml |   2 +
 21 files changed, 922 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d6e78bec/eagle-jpm/eagle-jpm-util/pom.xml
--
diff --git a/eagle-jpm/eagle-jpm-util/pom.xml b/eagle-jpm/eagle-jpm-util/pom.xml
index e424e49..575ea8c 100644
--- a/eagle-jpm/eagle-jpm-util/pom.xml
+++ b/eagle-jpm/eagle-jpm-util/pom.xml
@@ -68,5 +68,17 @@
 commons-codec
 1.9
 
+
+org.powermock
+powermock-module-junit4
+${powermock.version}
+test
+
+
+org.powermock
+powermock-api-mockito
+${powermock.version}
+test
+
 
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d6e78bec/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java
--
diff --git 
a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java
 
b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java
index f769217..6166a5d 100644
--- 
a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java
+++ 
b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java
@@ -207,4 +207,29 @@ public class AppInfo implements Serializable {
 this.runningContainers = runningContainers;
 }
 
+@Override
+public String toString() {
+return "AppInfo{"
++ "id='" + id + '\''
++ ", user='" + user + '\''
++ ", name='" + name + '\''
++ ", queue='" + queue + '\''
++ ", state='" + state + '\''
++ ", finalStatus='" + finalStatus + '\''
++ ", progress=" + progress
++ ", trackingUI='" + trackingUI + '\''
++ ", trackingUrl='" + trackingUrl + '\''
++ ", diagnostics='" + diagnostics + '\''
++ ", clusterId='" + clusterId + '\''
++ ", applicationType='" + applicationType + '\''
++ ", startedTime=" + startedTime
++ ", finishedTime=" + finishedTime
++ ", elapsedTime=" + elapsedTime
++ ", amContainerLogs='" + amContainerLogs + '\''
++ ", amHostHttpAddress='" + amHostHttpAddress + '\''
++ ", allocatedMB=" + allocatedMB
++ ", 

incubator-eagle git commit: [EAGLE-728] Application can't load application.properties when running local mode

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 6ec928941 -> ab3ed13e1


[EAGLE-728] Application can't load application.properties when running local 
mode

EAGLE-728 Application can't load application.properties when running local mode
- Use hard code, and overwrite it if user passes this configuration form web ui.

https://issues.apache.org/jira/browse/EAGLE-728

Author: chitin 

Closes #612 from chitin/EAGLE-728-2.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ab3ed13e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ab3ed13e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ab3ed13e

Branch: refs/heads/master
Commit: ab3ed13e18c85fa3fd64a08a070aaf24391aeb1e
Parents: 6ec9289
Author: chitin 
Authored: Wed Nov 9 10:47:06 2016 +0800
Committer: wujinhu 
Committed: Wed Nov 9 10:47:06 2016 +0800

--
 .../jpm/mr/history/parser/JHFEventReaderBase.java |  2 +-
 .../eagle/jpm/mr/running/MRRunningJobApplication.java |  3 ++-
 .../eagle/jpm/mr/running/parser/MRJobParser.java  |  8 ++--
 .../jpm/mr/running/storm/MRRunningJobParseBolt.java   |  8 ++--
 .../jpm/spark/history/crawl/JHFSparkEventReader.java  |  2 +-
 .../java/org/apache/eagle/jpm/util/Constants.java |  4 
 .../apache/eagle/jpm/util/JobNameNormalization.java   | 14 --
 7 files changed, 24 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
index e48370d..80cdb1c 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
@@ -200,7 +200,7 @@ public abstract class JHFEventReaderBase extends 
JobEntityCreationPublisher impl
 }
 
 if (jobDefId == null) {
-this.jobDefId = 
JobNameNormalization.getInstance().normalize(jobName);
+this.jobDefId = 
JobNameNormalization.getInstance(this.appConfig.getConfig()).normalize(jobName);
 } else {
 LOG.debug("Got JobDefId from job configuration for " + id + ": 
" + jobDefId);
 this.jobDefId = jobDefId;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
index bfdde13..e5c7c87 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
@@ -66,7 +66,8 @@ public class MRRunningJobApplication extends StormApplication 
{
 mrRunningJobConfig.getEagleServiceConfig(),
 mrRunningJobConfig.getEndpointConfig(),
 mrRunningJobConfig.getZkStateConfig(),
-confKeyKeys),
+confKeyKeys,
+config),
 tasks).setNumTasks(tasks).fieldsGrouping(spoutName, new 
Fields("appId"));
 return topologyBuilder.createTopology();
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 945e8ce..7b90dfc 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -18,6 +18,7 @@
 
 package org.apache.eagle.jpm.mr.running.parser;
 
+import com.typesafe.config.Config;
 import 

incubator-eagle git commit: [EAGLE-729] add hbase kerberose principle and keytab to provider

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 3595f29dc -> 725786cdb


[EAGLE-729] add hbase kerberose principle and keytab to provider

add hbase kerberose principle and keytab to provider

Author: yupu 

Closes #603 from puyulu/master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/725786cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/725786cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/725786cd

Branch: refs/heads/master
Commit: 725786cdb0626a73b63180c846c27521df30d53b
Parents: 3595f29
Author: yupu 
Authored: Thu Nov 3 18:04:25 2016 +0800
Committer: wujinhu 
Committed: Thu Nov 3 18:04:25 2016 +0800

--
 eagle.topology.TopologyCheckAppProvider.xml | 50 +++-
 1 file changed, 28 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/725786cd/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
--
diff --git 
a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
 
b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
index 8dbf79e..476b19c 100644
--- 
a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
+++ 
b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
@@ -22,13 +22,13 @@
 Topology Health Check
 0.5.0-incubating
 
-   
+
 
 dataExtractorConfig.site
 site
 Site
 sandbox
-  
+
 
 dataExtractorConfig.fetchDataIntervalInSecs
 Fetch Data Interval in Secs
@@ -52,15 +52,15 @@
 Bolt Task Number
 Bolt Task Number
 1
- 
-
+
+
 
 dataExtractorConfig.rackResolverCls
 Rack Resolver Class
 rack resolver class
 
org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver
- 
-  
+
+
 
 dataSourceConfig.hbase.zkQuorum
 HBase Zookeeper Quorum
@@ -88,14 +88,20 @@
 Hbase Master Principal
 hadoop/_h...@example.com
 
- 
+
 
-dataSourceConfig.hbase.kerberos.master.principal
-Hbase Master Principal
+dataSourceConfig.hbase.kerberos.eagle.principal
+Hbase Eagle Principal
 Hbase Master Principal
-hadoop/_h...@example.com
-  
-
+
+
+
+dataSourceConfig.hbase.kerberos.eagle.keytab
+Hbase Eagle KeyTab
+Hbase Eagle KeyTab
+
+
+
 
 dataSourceConfig.hdfs.namenodeUrl
 Hdfs Namenode Web URL
@@ -110,21 +116,20 @@
 http://sandbox.hortonworks.com:8088
 true
 
-
 
 dataSourceConfig.mr.historyServerUrl
 History Server URL
-History Server URL
-  
- 
-
+History Server URL
+
+
+
 
 topology.numOfSinkTasks
 topology.numOfSinkTasks
 2
 number of sink tasks
 
-
+
 
 
 dataSinkConfig.topic
@@ -150,14 +155,14 @@
 kafka.serializer.StringEncoder
 Serializer Class Kafka Message Key
 
-
+
 
 
 
 topology_health_check_stream
 topology health check Stream
 true
-true
+true
 
 
 status
@@ -170,7 +175,8 @@
 
 host
 string
- 
+
+
 
 site
 string
@@ -178,7 +184,7 @@
 
 role
 string
-
+
 
 
 

incubator-eagle git commit: [EAGLE-715] optimize some configurations for alert

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 6b0ed3d0d -> 56c2c4114


[EAGLE-715] optimize some configurations for alert

Author: wujinhu 

Closes #592 from wujinhu/EAGLE-715.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/56c2c411
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/56c2c411
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/56c2c411

Branch: refs/heads/master
Commit: 56c2c4114fc9dee8a1c2da47fa8045036f7d01bc
Parents: 6b0ed3d
Author: wujinhu 
Authored: Tue Nov 1 17:27:05 2016 +0800
Committer: wujinhu 
Committed: Tue Nov 1 17:27:05 2016 +0800

--
 history.MRHistoryJobApplicationProvider.xml | 23 ++--
 1 file changed, 12 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/56c2c411/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index 545cf56..835dbda 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -70,51 +70,52 @@
 
 
 dataSinkConfig.topic
-dataSinkConfig.topic
-map_reduce_failed_job
-topic for kafka data sink
+Destination(Kafka Topic) Of Stream Data
+sandbox_map_reduce_failed_job
+application emits stream data to this kafka 
topic
+true
 
 
 dataSinkConfig.brokerList
-dataSinkConfig.brokerList
+Broker(Kafka) List Of Stream Data
 localhost:6667
-kafka broker list
+kafka broker list for stream data
 true
 
 
 dataSinkConfig.serializerClass
-dataSinkConfig.serializerClass
+Serializer Class For Kafka Message Value
 kafka.serializer.StringEncoder
 serializer class Kafka message value
 
 
 dataSinkConfig.keySerializerClass
-dataSinkConfig.keySerializerClass
+Serializer Class For Kafka Message Key
 kafka.serializer.StringEncoder
 serializer class Kafka message key
 
 
 
 dataSinkConfig.producerType
-dataSinkConfig.producerType
+Producer Type Of Stream Data
 async
 whether the messages are sent asynchronously in a 
background thread
 
 
 dataSinkConfig.numBatchMessages
-dataSinkConfig.numBatchMessages
+Number Of Message Sent In One Batch
 4096
 number of messages to send in one batch when using 
async mode
 
 
 dataSinkConfig.maxQueueBufferMs
-dataSinkConfig.maxQueueBufferMs
+Maximum Time To Buffer Data
 5000
 maximum time to buffer data when using async 
mode
 
 
 dataSinkConfig.requestRequiredAcks
-dataSinkConfig.requestRequiredAcks
+Requires Ack For Stream Data
 0
 value controls when a produce request is considered 
completed
 

incubator-eagle git commit: [EAGLE-701] AlertEngine: bolt should check ZK for latest spec when start

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master e71d6d852 -> 7848680d3


[EAGLE-701] AlertEngine: bolt should check ZK for latest spec when start

Author: wujinhu 

Closes #586 from wujinhu/EAGLE-701.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7848680d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7848680d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7848680d

Branch: refs/heads/master
Commit: 7848680d362e3c14480149e8c100efdfcd6782dc
Parents: e71d6d8
Author: wujinhu 
Authored: Mon Oct 31 17:06:33 2016 +0800
Committer: wujinhu 
Committed: Mon Oct 31 17:06:33 2016 +0800

--
 .../apache/eagle/alert/config/ConfigBusConsumer.java  | 14 ++
 .../coordinator/IMetadataChangeNotifyService.java |  4 
 .../impl/ZKMetadataChangeNotifyService.java   |  5 +
 .../eagle/alert/engine/runner/AbstractStreamBolt.java |  5 +
 4 files changed, 24 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7848680d/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
index c2849a2..d5e6b4e 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
@@ -30,17 +30,17 @@ public class ConfigBusConsumer extends ConfigBusBase {
 private static final Logger LOG = 
org.slf4j.LoggerFactory.getLogger(ConfigBusConsumer.class);
 
 private NodeCache cache;
+private String zkPath;
+public static final ObjectMapper mapper = new ObjectMapper();
 
 public ConfigBusConsumer(ZKConfig config, String topic, 
ConfigChangeCallback callback) {
 super(config);
-String zkPath = zkRoot + "/" + topic;
+this.zkPath = zkRoot + "/" + topic;
 LOG.info("monitor change for zkPath " + zkPath);
 cache = new NodeCache(curator, zkPath);
 cache.getListenable().addListener(() -> {
 // get node value and notify callback
-byte[] value = curator.getData().forPath(zkPath);
-ObjectMapper mapper = new ObjectMapper();
-ConfigValue v = mapper.readValue(value, ConfigValue.class);
+ConfigValue v = getConfigValue();
 callback.onNewConfig(v);
 }
 );
@@ -51,4 +51,10 @@ public class ConfigBusConsumer extends ConfigBusBase {
 throw new RuntimeException(ex);
 }
 }
+
+public ConfigValue getConfigValue() throws Exception {
+byte[] value = curator.getData().forPath(zkPath);
+ConfigValue v = mapper.readValue(value, ConfigValue.class);
+return v;
+}
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7848680d/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
index 5aa754e..dab9f5a 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
@@ -50,4 +50,8 @@ public interface IMetadataChangeNotifyService extends 
Closeable, Serializable {
 void registerListener(StreamRouterBoltSpecListener listener);
 
 void registerListener(AlertPublishSpecListener listener);
+
+default void activateFetchMetaData() throws Exception {
+
+}
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7848680d/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/ZKMetadataChangeNotifyService.java

incubator-eagle git commit: [EAGLE-693] fix application could not detect stream change

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 4250e2d32 -> 4ff963b47


[EAGLE-693] fix application could not detect stream change

Author: wujinhu 

Closes #580 from wujinhu/EAGLE-700.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4ff963b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4ff963b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4ff963b4

Branch: refs/heads/master
Commit: 4ff963b47fd82cb1ca8394027518a9183f39176e
Parents: 4250e2d
Author: wujinhu 
Authored: Sat Oct 29 22:20:20 2016 +0800
Committer: wujinhu 
Committed: Sat Oct 29 22:20:20 2016 +0800

--
 .../eagle/app/service/ApplicationAction.java| 81 +++-
 .../src/main/bin/createTables.sql   | 20 +
 2 files changed, 65 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4ff963b4/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
index b7258d3..bd0adfe 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
@@ -88,45 +88,51 @@ public class ApplicationAction implements Serializable {
 }
 
 public void doInstall() {
-if (metadata.getDescriptor().getStreams() != null) {
-List streamDescToInstall = 
metadata.getDescriptor().getStreams().stream().map((streamDefinition -> {
-StreamDefinition copied = streamDefinition.copy();
-copied.setSiteId(metadata.getSite().getSiteId());
-
copied.setStreamId(StreamIdConversions.formatSiteStreamId(metadata.getSite().getSiteId(),
 copied.getStreamId()));
-StreamSinkConfig streamSinkConfig = this.runtime.environment()
+processStreams();
+}
+
+private void processStreams() {
+if (metadata.getDescriptor().getStreams() == null) {
+return;
+}
+
+List streamDescToInstall = 
metadata.getDescriptor().getStreams().stream().map((streamDefinition -> {
+StreamDefinition copied = streamDefinition.copy();
+copied.setSiteId(metadata.getSite().getSiteId());
+
copied.setStreamId(StreamIdConversions.formatSiteStreamId(metadata.getSite().getSiteId(),
 copied.getStreamId()));
+StreamSinkConfig streamSinkConfig = this.runtime.environment()
 
.streamSink().getSinkConfig(StreamIdConversions.parseStreamTypeId(copied.getSiteId(),
 copied.getStreamId()), this.effectiveConfig);
-StreamDesc streamDesc = new StreamDesc();
-streamDesc.setSchema(copied);
-streamDesc.setSink(streamSinkConfig);
-streamDesc.setStreamId(copied.getStreamId());
-return streamDesc;
-})).collect(Collectors.toList());
-metadata.setStreams(streamDescToInstall);
+StreamDesc streamDesc = new StreamDesc();
+streamDesc.setSchema(copied);
+streamDesc.setSink(streamSinkConfig);
+streamDesc.setStreamId(copied.getStreamId());
+return streamDesc;
+})).collect(Collectors.toList());
+metadata.setStreams(streamDescToInstall);
 
-// TODO: Decouple converting from StreamSink to Alert DataSource
-// iterate each stream descriptor and create alert datasource for 
each
-for (StreamDesc streamDesc : streamDescToInstall) {
-// only take care of Kafka sink
-if (streamDesc.getSink() instanceof KafkaStreamSinkConfig) {
-KafkaStreamSinkConfig kafkaCfg = (KafkaStreamSinkConfig) 
streamDesc.getSink();
-Kafka2TupleMetadata datasource = new Kafka2TupleMetadata();
-datasource.setType("KAFKA");
-datasource.setName(metadata.getAppId());
-datasource.setTopic(kafkaCfg.getTopicId());
-
datasource.setSchemeCls(JsonScheme.class.getCanonicalName());
-Tuple2StreamMetadata tuple2Stream = new 
Tuple2StreamMetadata();
-Properties prop = new Properties();
-
prop.put(JsonStringStreamNameSelector.USER_PROVIDED_STREAM_NAME_PROPERTY, 
streamDesc.getStreamId());
-

incubator-eagle git commit: [EAGLE-700] API /policies/{policyId} throws exception if policy does not exist

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master cbf6d3c10 -> 38d46c8bb


[EAGLE-700] API /policies/{policyId} throws exception if policy does not exist

Author: wujinhu 

Closes #579 from wujinhu/EAGLE-693.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/38d46c8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/38d46c8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/38d46c8b

Branch: refs/heads/master
Commit: 38d46c8bb549c9d6673c48f00a07bb82f57ad43a
Parents: cbf6d3c
Author: wujinhu 
Authored: Fri Oct 28 16:07:12 2016 +0800
Committer: wujinhu 
Committed: Fri Oct 28 16:07:12 2016 +0800

--
 .../service/metadata/resource/MetadataResource.java  | 11 ---
 1 file changed, 4 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/38d46c8b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java
index 05455f0..5110766 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java
@@ -306,12 +306,9 @@ public class MetadataResource {
 
 @Path("/policies/{policyId}")
 @GET
-public PolicyDefinition getPolicyByID(@PathParam("policyId") String 
policyId) {
-Preconditions.checkNotNull(policyId,"policyId");
-return dao.listPolicies().stream().filter(pc -> 
pc.getName().equals(policyId)).findAny().orElseGet(() -> {
-LOG.error("Policy (policyId " + policyId + ") not found");
-throw new IllegalArgumentException("Policy (policyId " + policyId 
+ ") not found");
-});
+public List getPolicyByID(@PathParam("policyId") String 
policyId) {
+Preconditions.checkNotNull(policyId, "policyId");
+return dao.listPolicies().stream().filter(pc -> 
pc.getName().equals(policyId)).collect(Collectors.toList());
 }
 
 @Path("/policies/{policyId}/status/{status}")
@@ -319,7 +316,7 @@ public class MetadataResource {
 public OpResult updatePolicyStatusByID(@PathParam("policyId") String 
policyId, @PathParam("status") PolicyDefinition.PolicyStatus status) {
 OpResult result = new OpResult();
 try {
-PolicyDefinition policyDefinition = getPolicyByID(policyId);
+PolicyDefinition policyDefinition = getPolicyByID(policyId).get(0);
 policyDefinition.setPolicyStatus(status);
 OpResult updateResult  = addPolicy(policyDefinition);
 result.code = updateResult.code;

incubator-eagle git commit: [EAGLE-694] alert engine could not reduce alert bolt number when parallelism of p…

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master ab1c9b64f -> c4a0b94d7


[EAGLE-694] alert engine could not reduce alert bolt number when parallelism of 
p…

Author: wujinhu 

Closes #577 from wujinhu/EAGLE-694.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c4a0b94d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c4a0b94d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c4a0b94d

Branch: refs/heads/master
Commit: c4a0b94d760b54b3613e11d67e355e982a18910f
Parents: ab1c9b6
Author: wujinhu 
Authored: Fri Oct 28 14:54:51 2016 +0800
Committer: wujinhu 
Committed: Fri Oct 28 14:54:51 2016 +0800

--
 .../alert/engine/coordinator/PolicyDefinition.java |  2 +-
 .../coordinator/provider/ScheduleContextBuilder.java   |  4 ++--
 .../alert/coordinator/ScheduleContextBuilderTest.java  | 13 +++--
 3 files changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c4a0b94d/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
index 6df682a..cfd7fef 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
@@ -153,7 +153,7 @@ public class PolicyDefinition implements Serializable {
 && (another.definition != null && 
another.definition.equals(this.definition))
 && Objects.equals(this.definition, another.definition)
 && CollectionUtils.isEqualCollection(another.partitionSpec, 
this.partitionSpec)
-// && another.parallelismHint == this.parallelismHint
+ && another.parallelismHint == this.parallelismHint
 ) {
 return true;
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c4a0b94d/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
index 69225da..98b598a 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
@@ -224,8 +224,8 @@ public class ScheduleContextBuilder {
 } else {
 StreamWorkSlotQueue queue = 
queueMap.get(assignment.getQueueId());
 if (queue == null
-|| 
policies.get(assignment.getPolicyName()).getParallelismHint() > 
queue.getQueueSize()) {
-// queue not found or policy has hint bigger than queue 
(possible a poilcy update)
+|| 
policies.get(assignment.getPolicyName()).getParallelismHint() != 
queue.getQueueSize()) {
+// queue not found or policy has hint not equal to queue 
(possible a poilcy update)
 LOG.info("Policy assignment {} 's policy doesnt match 
queue: {}!", assignment, queue);
 paIt.remove();
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c4a0b94d/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java
index ac83c73..e7efbd7 100644
--- 

incubator-eagle git commit: [EAGLE-674] fix grouping between AlertBolt and AlertPublisher

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master c15e7f814 -> 8a34294ec


[EAGLE-674] fix grouping between AlertBolt and AlertPublisher

Author: wujinhu 

Closes #557 from wujinhu/EAGLE-673.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8a34294e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8a34294e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8a34294e

Branch: refs/heads/master
Commit: 8a34294ec604b42bee5dd99f40ba6e9f9fc498b5
Parents: c15e7f8
Author: wujinhu 
Authored: Mon Oct 24 17:21:37 2016 +0800
Committer: wujinhu 
Committed: Mon Oct 24 17:21:37 2016 +0800

--
 .../org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8a34294e/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
index 287d5db..3b40afd 100755
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
@@ -205,7 +205,8 @@ public class UnitTopologyRunner {
 // connect alert bolt and alert publish bolt, this is the last bolt in 
the pipeline
 BoltDeclarer boltDeclarer = builder.setBolt(alertPublishBoltName, 
publisherBolt, numOfPublishExecutors).setNumTasks(numOfPublishTasks);
 for (int i = 0; i < numOfAlertBolts; i++) {
-boltDeclarer.fieldsGrouping(alertBoltNamePrefix + i, new 
Fields(AlertConstants.FIELD_0));
+//boltDeclarer.fieldsGrouping(alertBoltNamePrefix + i, new 
Fields(AlertConstants.FIELD_0));
+boltDeclarer.shuffleGrouping(alertBoltNamePrefix + i);
 }
 
 return builder.createTopology();

incubator-eagle git commit: [EAGLE-673] add numOfPublishExecutors to alert engine topology

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 56e7048ff -> e520e4011


[EAGLE-673] add numOfPublishExecutors to alert engine topology

Author: wujinhu 

Closes #555 from wujinhu/EAGLE-673.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e520e401
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e520e401
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e520e401

Branch: refs/heads/master
Commit: e520e4011796ebdea52f80ca43b9bffddf3aa50a
Parents: 56e7048
Author: wujinhu 
Authored: Mon Oct 24 14:09:43 2016 +0800
Committer: wujinhu 
Committed: Mon Oct 24 14:09:43 2016 +0800

--
 ...agle.alert.app.AlertUnitTopologyAppProvider.xml |  7 +++
 .../src/main/resources/application.conf|  1 +
 .../alert/engine/runner/UnitTopologyRunner.java| 17 -
 3 files changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e520e401/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
 
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
index 6ef96c7..28f7db4 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
@@ -51,6 +51,13 @@
 false
 
 
+topology.numOfPublishExecutors
+Publisher Executor Number
+1
+Number of publish executors
+false
+
+
 topology.numOfPublishTasks
 Publisher Tasks Number
 1

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e520e401/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
 
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
index 1a25cfa..46f5b08 100644
--- 
a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
@@ -22,6 +22,7 @@
 "numOfSpoutTasks" : 1,
 "numOfRouterBolts" : 4,
 "numOfAlertBolts" : 10,
+"numOfPublishExecutors" : 1,
 "numOfPublishTasks" : 1,
 "messageTimeoutSecs": 3600,
 "localMode" : "true"

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e520e401/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
--
diff --git 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
index 88cfb9b..287d5db 100755
--- 
a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
+++ 
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
@@ -63,6 +63,7 @@ public class UnitTopologyRunner {
 public static final String ROUTER_TASK_NUM = "topology.numOfRouterBolts";
 public static final String ALERT_TASK_NUM = "topology.numOfAlertBolts";
 public static final String PUBLISH_TASK_NUM = "topology.numOfPublishTasks";
+public static final String PUBLISH_EXECUTOR_NUM = 
"topology.numOfPublishExecutors";
 public static final String LOCAL_MODE = "topology.localMode";
 public static final String MESSAGE_TIMEOUT_SECS = 
"topology.messageTimeoutSecs";
 public static final int DEFAULT_MESSAGE_TIMEOUT_SECS = 3600;
@@ -88,6 +89,7 @@ public class UnitTopologyRunner {
  int numOfSpoutTasks,
  int numOfRouterBolts,
  int numOfAlertBolts,
+ int numOfPublishExecutors,
  int numOfPublishTasks,
 

incubator-eagle git commit: [EAGLE-662] remove jarPath/mode from entity context

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 3980dcb4c -> 8d662e3a2


[EAGLE-662] remove jarPath/mode from entity context

Author: wujinhu 

Closes #550 from wujinhu/EAGLE-662.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8d662e3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8d662e3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8d662e3a

Branch: refs/heads/master
Commit: 8d662e3a2ee4fc4411fa903cb5005cd81a65aba5
Parents: 3980dcb
Author: wujinhu 
Authored: Sat Oct 22 14:07:51 2016 +0800
Committer: wujinhu 
Committed: Sat Oct 22 14:07:51 2016 +0800

--
 .../java/org/apache/eagle/app/service/ApplicationAction.java   | 6 +-
 .../app/service/impl/ApplicationManagementServiceImpl.java | 2 --
 .../org/apache/eagle/app/storm/MockStormApplicationTest.java   | 3 +++
 .../eagle/app/example/ExampleApplicationProviderTest.java  | 4 
 4 files changed, 12 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8d662e3a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
index 0b389b6..557737f 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
@@ -40,6 +40,7 @@ import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.Properties;
 import java.util.stream.Collectors;
 
@@ -70,13 +71,16 @@ public class ApplicationAction implements Serializable {
 this.application = application;
 this.metadata = metadata;
 this.runtime = 
ExecutionRuntimeManager.getInstance().getRuntime(application.getEnvironmentType(),
 serverConfig);
-Map executionConfig = metadata.getConfiguration();
+Map executionConfig = new 
HashMap<>(metadata.getConfiguration());
 if (executionConfig == null) {
 executionConfig = Collections.emptyMap();
 }
 if (serverConfig.hasPath(MetricConfigs.METRIC_PREFIX_CONF)) {
 LOG.warn("Ignored sever config {} = {}", 
MetricConfigs.METRIC_PREFIX_CONF, 
serverConfig.getString(MetricConfigs.METRIC_PREFIX_CONF));
 }
+
+executionConfig.put("jarPath", metadata.getJarPath());
+executionConfig.put("mode", metadata.getMode().name());
 executionConfig.put(MetricConfigs.METRIC_PREFIX_CONF, 
APP_METRIC_PREFIX);
 this.effectiveConfig = 
ConfigFactory.parseMap(executionConfig).withFallback(serverConfig).withFallback(ConfigFactory.parseMap(metadata.getContext()));
 this.alertMetadataService = alertMetadataService;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8d662e3a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
index 6163a8f..003f966 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
@@ -101,9 +101,7 @@ public class ApplicationManagementServiceImpl implements 
ApplicationManagementSe
 applicationEntity.setConfiguration(appConfig);
 
 applicationEntity.getContext().put("siteId", siteEntity.getSiteId());
-applicationEntity.getContext().put("mode", 
applicationEntity.getMode().name());
 applicationEntity.getContext().put("appId", 
applicationEntity.getAppId());
-applicationEntity.getContext().put("jarPath", 
applicationEntity.getJarPath());
 
 // Validate Dependency
 validateDependingApplicationInstalled(applicationEntity);

incubator-eagle git commit: [EAGLE-655] fix metadata bug

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 823c66dae -> 538e88f71


[EAGLE-655] fix metadata bug

Author: wujinhu 

Closes #545 from wujinhu/EAGLE-653.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/538e88f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/538e88f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/538e88f7

Branch: refs/heads/master
Commit: 538e88f71c0cf8011dc4709b62b3e45b04ac1982
Parents: 823c66d
Author: wujinhu 
Authored: Thu Oct 20 21:36:49 2016 +0800
Committer: wujinhu 
Committed: Thu Oct 20 21:36:49 2016 +0800

--
 .../eagle/metadata/model/ApplicationEntity.java |  6 ++--
 .../ApplicationEntityServiceJDBCImpl.java   |  5 ++-
 .../orm/ApplicationEntityToRelation.java|  4 +++
 .../orm/RelationToApplicationEntity.java| 32 
 .../ApplicationEntityServiceJDBCImplTest.java   |  3 ++
 .../src/test/resources/init.sql |  1 +
 .../src/main/bin/createTables.sql   |  1 +
 7 files changed, 36 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/538e88f7/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
--
diff --git 
a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
 
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
index 952b5a8..920e6c9 100644
--- 
a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
+++ 
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
@@ -35,7 +35,7 @@ public class ApplicationEntity extends PersistenceEntity {
 private ApplicationDesc descriptor;
 
 private Map configuration = new HashMap<>();
-private Map context = new HashMap<>();
+private Map context = new HashMap<>();
 private List streams;
 private Mode mode = Mode.CLUSTER;
 private String jarPath;
@@ -118,11 +118,11 @@ public class ApplicationEntity extends PersistenceEntity {
 }
 }
 
-public Map getContext() {
+public Map getContext() {
 return context;
 }
 
-public void setContext(Map context) {
+public void setContext(Map context) {
 this.context = context;
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/538e88f7/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java
--
diff --git 
a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java
 
b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java
index 1f5ba65..96b33bf 100644
--- 
a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java
+++ 
b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java
@@ -43,7 +43,7 @@ public class ApplicationEntityServiceJDBCImpl implements 
ApplicationEntityServic
 
 private static final Logger LOGGER = 
LoggerFactory.getLogger(ApplicationEntityServiceJDBCImpl.class);
 
-private static final String insertSql = "INSERT INTO applications (siteid, 
apptype, appmode, jarpath, appstatus, configuration, createdtime, modifiedtime, 
uuid, appid ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+private static final String insertSql = "INSERT INTO applications (siteid, 
apptype, appmode, jarpath, appstatus, configuration, context, createdtime, 
modifiedtime, uuid, appid ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
 private static final String selectSql = "SELECT * FROM applications a 
INNER JOIN sites s on  a.siteid = s.siteid";
 private static final String selectSqlBySiteIdAndAppType = "SELECT * FROM 
applications  a INNER JOIN sites s on  a.siteid = s.siteid where a.siteid = ? 
and a.apptype = ?";
 private static final String selectSqlBySiteId = "SELECT * FROM 
applications  a INNER JOIN sites s on  a.siteid = s.siteid where a.siteid = ?";
@@ -145,6 +145,9 @@ public class 

incubator-eagle git commit: [EAGLE-638] clean up configuration for Eagle JPM aggregation feeder

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 6f88c30f8 -> c45ac6d57


[EAGLE-638] clean up configuration for Eagle JPM aggregation feeder

Author: wujinhu 

Closes #533 from wujinhu/EAGLE-638.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c45ac6d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c45ac6d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c45ac6d5

Branch: refs/heads/master
Commit: c45ac6d57a29cecde179476b93ca53fd2ef89cc1
Parents: 6f88c30
Author: wujinhu 
Authored: Wed Oct 19 17:15:49 2016 +0800
Committer: wujinhu 
Committed: Wed Oct 19 17:15:49 2016 +0800

--
 .../jpm/aggregation/AggregationApplication.java | 19 ++---
 .../jpm/aggregation/AggregationConfig.java  | 49 ++-
 .../jpm/aggregation/mr/MRMetricAggregator.java  |  2 +-
 .../mr/MRMetricsAggregateContainer.java |  2 +-
 .../jpm/aggregation/storm/AggregationBolt.java  |  2 +-
 .../jpm/aggregation/storm/AggregationSpout.java |  2 +-
 ...gregation.AggregationApplicationProvider.xml | 85 +++-
 ...org.apache.eagle.app.spi.ApplicationProvider | 16 
 .../src/main/resources/application.conf | 34 +++-
 eagle.app.jpm.JPMWebApplicationProvider.xml |  2 -
 eagle-server/pom.xml|  5 ++
 11 files changed, 92 insertions(+), 126 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c45ac6d5/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
--
diff --git 
a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
 
b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
index 214c688..750c9ab 100644
--- 
a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
+++ 
b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
@@ -31,11 +31,10 @@ import java.util.*;
 public class AggregationApplication extends StormApplication {
 @Override
 public StormTopology execute(Config config, StormEnvironment environment) {
-//TODO
 List metricNames = new ArrayList<>();
 String[] metricNamesArr = 
config.getString("aggregate.counters.metrics").split(",");
 for (int i = 0; i < metricNamesArr.length; i++) {
-metricNames.add(metricNamesArr[i].trim());
+metricNames.add(metricNamesArr[i].trim().toLowerCase());
 }
 List groupByColumns = new ArrayList<>();
 
@@ -56,25 +55,17 @@ public class AggregationApplication extends 
StormApplication {
 String spoutName = "mrHistoryAggregationSpout";
 String boltName = "mrHistoryAggregationBolt";
 AggregationConfig aggregationConfig = 
AggregationConfig.getInstance(config);
-int parallelism = 
aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + 
spoutName);
-int tasks = 
aggregationConfig.getConfig().getInt("envContextConfig.tasks." + spoutName);
-if (parallelism > tasks) {
-parallelism = tasks;
-}
+int tasks = aggregationConfig.getConfig().getInt("stormConfig." + 
spoutName + "Tasks");
 topologyBuilder.setSpout(
 spoutName,
 new AggregationSpout(config, new 
MRMetricsAggregateContainer(metrics)),
-parallelism
+tasks
 ).setNumTasks(tasks);
 
-parallelism = 
aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + 
boltName);
-tasks = aggregationConfig.getConfig().getInt("envContextConfig.tasks." 
+ boltName);
-if (parallelism > tasks) {
-parallelism = tasks;
-}
+tasks = aggregationConfig.getConfig().getInt("stormConfig." + boltName 
+ "Tasks");
 topologyBuilder.setBolt(boltName,
 new AggregationBolt(config, new 
MRMetricsAggregateContainer(metrics)),
-parallelism).setNumTasks(tasks).shuffleGrouping(spoutName);
+tasks).setNumTasks(tasks).shuffleGrouping(spoutName);
 
 return topologyBuilder.createTopology();
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c45ac6d5/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
--
diff --git 
a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
 

incubator-eagle git commit: [EAGLE-633] MR history feeder read some configurations from eagle server

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 310935d27 -> 2479d3279


[EAGLE-633] MR history feeder read some configurations from eagle server

Author: wujinhu 

Closes #525 from wujinhu/EAGLE-633.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2479d327
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2479d327
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2479d327

Branch: refs/heads/master
Commit: 2479d32791dd1a424af7513279ba28543baf6af4
Parents: 310935d
Author: wujinhu 
Authored: Tue Oct 18 17:20:37 2016 +0800
Committer: wujinhu 
Committed: Tue Oct 18 17:20:37 2016 +0800

--
 .../jpm/mr/history/MRHistoryJobConfig.java  | 20 +++---
 history.MRHistoryJobApplicationProvider.xml | 64 +---
 .../src/main/resources/application.conf |  4 +-
 eagle-server-assembly/src/main/conf/eagle.conf  | 10 +++
 .../src/main/resources/application.conf | 10 +++
 5 files changed, 33 insertions(+), 75 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2479d327/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
index 561c3d0..566f7d0 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
@@ -119,11 +119,11 @@ public class MRHistoryJobConfig implements Serializable {
 this.config = config;
 
 //parse eagle zk
-this.zkStateConfig.zkQuorum = 
config.getString("zkStateConfig.zkQuorum");
-this.zkStateConfig.zkSessionTimeoutMs = 
config.getInt("zkStateConfig.zkSessionTimeoutMs");
-this.zkStateConfig.zkRetryTimes = 
config.getInt("zkStateConfig.zkRetryTimes");
-this.zkStateConfig.zkRetryInterval = 
config.getInt("zkStateConfig.zkRetryInterval");
-this.zkStateConfig.zkRoot = config.getString("zkStateConfig.zkRoot");
+this.zkStateConfig.zkQuorum = config.getString("zookeeper.zkQuorum");
+this.zkStateConfig.zkSessionTimeoutMs = 
config.getInt("zookeeper.zkSessionTimeoutMs");
+this.zkStateConfig.zkRetryTimes = 
config.getInt("zookeeper.zkRetryTimes");
+this.zkStateConfig.zkRetryInterval = 
config.getInt("zookeeper.zkRetryInterval");
+this.zkStateConfig.zkRoot = config.getString("zookeeper.zkRoot");
 
 //parse job history endpoint
 this.jobHistoryEndpointConfig.site = config.getString("siteId");
@@ -135,12 +135,12 @@ public class MRHistoryJobConfig implements Serializable {
 this.jobHistoryEndpointConfig.timeZone = 
config.getString("endpointConfig.timeZone");
 
 // parse eagle service endpoint
-this.eagleServiceConfig.eagleServiceHost = 
config.getString("eagleService.host");
-String port = config.getString("eagleService.port");
+this.eagleServiceConfig.eagleServiceHost = 
config.getString("service.host");
+String port = config.getString("service.port");
 this.eagleServiceConfig.eagleServicePort = (port == null ? 8080 : 
Integer.parseInt(port));
-this.eagleServiceConfig.username = 
config.getString("eagleService.username");
-this.eagleServiceConfig.password = 
config.getString("eagleService.password");
-this.eagleServiceConfig.readTimeoutSeconds = 
config.getInt("eagleService.readTimeOutSeconds");
+this.eagleServiceConfig.username = 
config.getString("service.username");
+this.eagleServiceConfig.password = 
config.getString("service.password");
+this.eagleServiceConfig.readTimeoutSeconds = 
config.getInt("service.readTimeOutSeconds");
 
 LOG.info("Successfully initialized MRHistoryJobConfig");
 LOG.info("zkStateConfig.zkQuorum: " + this.zkStateConfig.zkQuorum);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2479d327/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
 

incubator-eagle git commit: [EAGLE-632] clean up configuration for MR history feeder

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 3d6a29ec2 -> a781937e9


[EAGLE-632] clean up configuration for MR history feeder

Author: wujinhu 

Closes #522 from wujinhu/EAGLE-632.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a781937e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a781937e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a781937e

Branch: refs/heads/master
Commit: a781937e978025b708ccc70d47ddaadc18668948
Parents: 3d6a29e
Author: wujinhu 
Authored: Tue Oct 18 15:43:44 2016 +0800
Committer: wujinhu 
Committed: Tue Oct 18 15:43:44 2016 +0800

--
 .../jpm/mr/history/MRHistoryJobApplication.java |  14 +-
 .../jpm/mr/history/MRHistoryJobConfig.java  |  62 +---
 .../crawler/DefaultJHFInputStreamCallback.java  |   2 +-
 .../history/crawler/JHFCrawlerDriverImpl.java   |   6 +-
 .../metrics/JobCountMetricsGenerator.java   |   2 +-
 .../mr/history/parser/JHFEventReaderBase.java   |  20 +-
 .../mr/history/parser/JHFMRVer1EventReader.java | 152 --
 .../jpm/mr/history/parser/JHFMRVer1Parser.java  | 283 ---
 .../parser/JHFMRVer1PerLineListener.java|  39 ---
 .../mr/history/parser/JHFMRVer2EventReader.java |   1 -
 .../jpm/mr/history/parser/JHFParserFactory.java |  44 +--
 ...JobConfigurationCreationServiceListener.java |   2 +-
 .../JobEntityCreationEagleServiceListener.java  |   4 +-
 .../parser/TaskAttemptCounterListener.java  |   2 +-
 .../mr/history/parser/TaskFailureListener.java  |   2 +-
 .../jpm/mr/history/storm/JobHistorySpout.java   |  15 +-
 history.MRHistoryJobApplicationProvider.xml | 102 ---
 .../src/main/resources/application.conf |  44 +--
 18 files changed, 131 insertions(+), 665 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a781937e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
index beec938..de35678 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
@@ -59,16 +59,12 @@ public class MRHistoryJobApplication extends 
StormApplication {
 JobHistoryContentFilter filter = builder.build();
 //3. init topology
 TopologyBuilder topologyBuilder = new TopologyBuilder();
-String spoutName = "mrHistoryJobExecutor";
-int parallelism = 
jhfAppConf.getInt("envContextConfig.parallelismConfig." + spoutName);
-int tasks = jhfAppConf.getInt("envContextConfig.tasks." + spoutName);
-if (parallelism > tasks) {
-parallelism = tasks;
-}
+String spoutName = "mrHistoryJobSpout";
+int tasks = jhfAppConf.getInt("stormConfig.mrHistoryJobSpoutTasks");
 topologyBuilder.setSpout(
-spoutName,
-new JobHistorySpout(filter, config),
-parallelism
+spoutName,
+new JobHistorySpout(filter, config),
+tasks
 ).setNumTasks(tasks);
 return topologyBuilder.createTopology();
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a781937e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
index 37df5ad..561c3d0 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
@@ -49,18 +49,6 @@ public class MRHistoryJobConfig implements Serializable {
 
 private JobHistoryEndpointConfig jobHistoryEndpointConfig;
 
-public ControlConfig getControlConfig() {
-return controlConfig;
-}
-
-private ControlConfig controlConfig;
-
-public JobExtractorConfig getJobExtractorConfig() {
-return jobExtractorConfig;
-}
-
-private JobExtractorConfig jobExtractorConfig;
-
 public EagleServiceConfig getEagleServiceConfig() {
 

incubator-eagle git commit: [EAGLE-628] Fix jdbc metadata bugs

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master ecda7d5ab -> 473496747


[EAGLE-628] Fix jdbc metadata bugs

https://issues.apache.org/jira/browse/EAGLE-628

Author: wujinhu 

Closes #520 from wujinhu/master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/47349674
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/47349674
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/47349674

Branch: refs/heads/master
Commit: 473496747d42b9bfe72f32cb89c312eef1a360bf
Parents: ecda7d5
Author: wujinhu 
Authored: Mon Oct 17 19:42:23 2016 +0800
Committer: wujinhu 
Committed: Mon Oct 17 19:42:23 2016 +0800

--
 .../app/module/ApplicationGuiceModule.java  |   8 +-
 .../eagle-metadata/eagle-metadata-jdbc/pom.xml  |   5 +
 .../ApplicationEntityServiceJDBCImpl.java   |  75 +++--
 .../jdbc/service/SiteEntityServiceJDBCImpl.java |  16 +--
 .../orm/ApplicationEntityToRelation.java|   5 +
 .../orm/RelationToApplicationEntity.java|  36 --
 .../ApplicationEntityServiceJDBCImplTest.java   |  40 ---
 .../store/jdbc/JDBCDataSourceProviderTest.java  |   2 +-
 .../jdbc/JDBCMetadataQueryServiceTest.java  |   2 +-
 .../store/jdbc/JDBCMetadataTestBase.java| 109 ++-
 .../store/jdbc/TestStaticApplication.java   |  31 ++
 ...tore.jdbc.TestStaticApplication$Provider.xml |  31 ++
 ...org.apache.eagle.app.spi.ApplicationProvider |  16 +++
 .../src/test/resources/init.sql |   7 +-
 .../service/JDBCSecurityMetadataDAO.java|  12 +-
 .../audit/JDBCSecurityMetadataDAOTest.java  |  11 +-
 .../src/main/bin/createTables.sql   |  40 +++
 17 files changed, 340 insertions(+), 106 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/47349674/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java
--
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java
index 67c7eec..d796faa 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java
@@ -16,9 +16,11 @@
  */
 package org.apache.eagle.app.module;
 
+import com.typesafe.config.ConfigFactory;
 import org.apache.eagle.app.service.ApplicationManagementService;
 import org.apache.eagle.app.service.ApplicationProviderService;
 import org.apache.eagle.app.service.impl.ApplicationManagementServiceImpl;
+import org.apache.eagle.app.service.impl.ApplicationProviderServiceImpl;
 import org.apache.eagle.app.service.impl.ApplicationStatusUpdateServiceImpl;
 import org.apache.eagle.metadata.service.ApplicationDescService;
 import com.google.inject.AbstractModule;
@@ -27,12 +29,16 @@ import com.google.inject.util.Providers;
 import org.apache.eagle.metadata.service.ApplicationStatusUpdateService;
 
 public class ApplicationGuiceModule extends AbstractModule {
-private ApplicationProviderService appProviderInst;
+private final ApplicationProviderService appProviderInst;
 
 public ApplicationGuiceModule(ApplicationProviderService appProviderInst) {
 this.appProviderInst = appProviderInst;
 }
 
+public ApplicationGuiceModule() {
+this.appProviderInst = new 
ApplicationProviderServiceImpl(ConfigFactory.load());
+}
+
 @Override
 protected void configure() {
 
bind(ApplicationProviderService.class).toProvider(Providers.of(appProviderInst));

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/47349674/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml
--
diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml 
b/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml
index 940d697..d15f30a 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml
+++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml
@@ -46,5 +46,10 @@
 com.h2database
 h2
 
+
+org.apache.eagle
+eagle-app-base
+${project.version}
+
 
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/47349674/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java

incubator-eagle git commit: [EAGLE-618] migration eagle-jpm-aggregation to application framework-bug fix

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master e6f9d1bbd -> c6ac2eb6f


[EAGLE-618] migration eagle-jpm-aggregation to application framework-bug fix

Author: wujinhu 

Closes #508 from wujinhu/EAGLE-618.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c6ac2eb6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c6ac2eb6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c6ac2eb6

Branch: refs/heads/master
Commit: c6ac2eb6f1b5ba6d6c8702e79e7f669e77dcb0a7
Parents: e6f9d1b
Author: wujinhu 
Authored: Fri Oct 14 15:50:39 2016 +0800
Committer: wujinhu 
Committed: Fri Oct 14 15:50:39 2016 +0800

--
 .../apache/eagle/jpm/aggregation/AggregationApplication.java| 5 +++--
 ...che.eagle.jpm.aggregation.AggregationApplicationProvider.xml | 4 ++--
 .../eagle-jpm-aggregation/src/main/resources/application.conf   | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c6ac2eb6/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
--
diff --git 
a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
 
b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
index 0577070..b040d3e 100644
--- 
a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
+++ 
b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
@@ -38,7 +38,8 @@ public class AggregationApplication extends StormApplication {
 metricNames.add(metricNamesArr[i]);
 }
 List groupByColumns = new ArrayList<>();
-String[] groupByColumnsArr = 
config.getString("aggregate.counters.groupBys").split(",");
+
+String[] groupByColumnsArr = 
config.getString("aggregate.counters.groupBys").split(";");
 for (int i = 0; i < groupByColumnsArr.length; i++) {
 groupByColumns.add(groupByColumnsArr[i]);
 }
@@ -47,7 +48,7 @@ public class AggregationApplication extends StormApplication {
 for (String metric : metricNames) {
 metrics.put(metric, new ArrayList<>());
 for (String cols : groupByColumns) {
-metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", 
"").split("&")));
+metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", 
"").split(",")));
 }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c6ac2eb6/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
--
diff --git 
a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
 
b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
index 237a437..99e3fca 100644
--- 
a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
+++ 
b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
@@ -102,8 +102,8 @@
 
 
 aggregate.counters.groupBys
-groupBys that each metric need to aggregate. If group 
by many columns, then split them by &
-site, site, site
+groupBys that each metric need to aggregate. If group 
by many columns, then split them by ,
+site, jobType; site, user; site
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c6ac2eb6/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
--
diff --git 
a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf 
b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
index 98fc009..529dd57 100644
--- a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
@@ -51,7 +51,7 @@
   "aggregate" : {
 "counters" : {
   "metrics" : "cpu_milliseconds,file_bytes_read, file_bytes_written, 
hdfs_bytes_read, hdfs_bytes_written, hdfs_read_ops, hdfs_write_ops",
-  "groupBys": "site, site, site"
+  "groupBys": 

incubator-eagle git commit: [EAGLE-539] add failure category and job count by state to mr feeder

[jinhuwu]

Repository: incubator-eagle
Updated Branches:
  refs/heads/master 02aab65b7 -> 83bb1


[EAGLE-539] add failure category and job count by state to mr feeder

https://issues.apache.org/jira/browse/EAGLE-539

1. We need to classify the errors that generated by mr jobs so that we can 
analysis requirements like top-n errors or top-n hosts that generate errors. I 
implement this by using the error messages that generated by the tasks and 
extract the rules from them.
2. Another requirement is get job count by job status. When we parse jobs in mr 
history feeder, we can save job id and job status in zookeeper, and then we can 
flush them to eagle server.

Author: wujinhu 

Closes #434 from wujinhu/EAGLE-539.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/83bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/83bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/83bb

Branch: refs/heads/master
Commit: 83bb1b46e8d69497e9459d7cca78264a1300
Parents: 02aab65
Author: wujinhu 
Authored: Mon Sep 19 18:04:17 2016 +0800
Committer: wujinhu 
Committed: Mon Sep 19 18:04:17 2016 +0800

--
 .../mr/historyentity/JobExecutionAPIEntity.java | 13 ++
 .../metrics/JobCountMetricsGenerator.java   | 43 ++--
 .../JobExecutionMetricsCreationListener.java| 10 +
 .../mr/history/parser/JHFEventReaderBase.java   | 33 +++
 .../mr/history/parser/TaskFailureListener.java  |  1 +
 .../jpm/mr/running/parser/MRJobParser.java  |  3 +-
 .../org/apache/eagle/jpm/util/Constants.java|  3 +-
 .../java/org/apache/eagle/jpm/util/Utils.java   | 26 
 8 files changed, 99 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/83bb/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
--
diff --git 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
index f7540d5..0e40099 100644
--- 
a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
+++ 
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
@@ -23,6 +23,8 @@ import org.apache.eagle.jpm.util.jobcounter.JobCounters;
 import org.apache.eagle.log.entity.meta.*;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
+import java.util.Map;
+
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 @Table("eaglejpa")
 @ColumnFamily("f")
@@ -91,6 +93,8 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity {
 private int failedReduceAttempts;
 @Column("ad")
 private String trackingUrl;
+@Column("ae")
+private Map> failedTasks;
 
 public String getTrackingUrl() {
 return trackingUrl;
@@ -343,4 +347,13 @@ public class JobExecutionAPIEntity extends 
JobBaseAPIEntity {
 this.failedReduceAttempts = failedReduceAttempts;
 valueChanged("failedReduceAttempts");
 }
+
+public Map> getFailedTasks() {
+return failedTasks;
+}
+
+public void setFailedTasks(Map> failedTasks) {
+this.failedTasks = failedTasks;
+valueChanged("failedTasks");
+}
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/83bb/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/metrics/JobCountMetricsGenerator.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/metrics/JobCountMetricsGenerator.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/metrics/JobCountMetricsGenerator.java
index a6f9d56..ac4a33d 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/metrics/JobCountMetricsGenerator.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/metrics/JobCountMetricsGenerator.java
@@ -22,6 +22,7 @@ import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig;
 import org.apache.eagle.jpm.mr.history.parser.EagleJobStatus;
 import org.apache.eagle.jpm.mr.history.zkres.JobHistoryZKStateManager;
 import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.jpm.util.MRJobTagName;
 import org.apache.eagle.log.entity.GenericMetricEntity;
 import