incubator-eagle git commit: [EAGLE-834] Update Daily Job Summery Report

2016-12-15 Thread qingwzhao 
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 229d7b907 -> a89275bfc


[EAGLE-834] Update Daily Job Summery Report

https://issues.apache.org/jira/browse/EAGLE-834

* update mail subject & title
* adjust mail template and enrich the summary data
* fix a query bug in the mr job list API (GET /mrJobs)

Author: Zhao, Qingwen 

Closes #749 from qingwen220/EAGLE-834.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a89275bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a89275bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a89275bf

Branch: refs/heads/master
Commit: a89275bfc86c4db63688e28ff12196ee97a78207
Parents: 229d7b9
Author: Zhao, Qingwen 
Authored: Fri Dec 16 15:04:00 2016 +0800
Committer: Zhao, Qingwen 
Committed: Fri Dec 16 15:04:00 2016 +0800

--
 .../mr/history/MRHistoryJobDailyReporter.java   | 169 +--
 .../src/main/resources/JobReportTemplate.vm | 118 +++--
 .../history/MRHistoryJobDailyReporterTest.java  |  43 ++---
 .../service/jpm/MRJobExecutionResource.java |  26 +--
 4 files changed, 177 insertions(+), 179 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
index 6b38244..0dc6c5f 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
@@ -39,19 +39,20 @@ import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
+import static 
org.apache.eagle.common.config.EagleConfigConstants.EAGLE_TIME_ZONE;
 import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_HOST;
 import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_PORT;
 
 public class MRHistoryJobDailyReporter extends AbstractScheduledService {
 private static final Logger LOG = 
LoggerFactory.getLogger(MRHistoryJobDailyReporter.class);
-private static final String TIMEZONE_PATH = "service.timezone";
+
 private static final String DAILY_SENT_HOUROFDAY = 
"application.dailyJobReport.reportHourTime";
 private static final String DAILY_SENT_PERIOD = 
"application.dailyJobReport.reportPeriodInHour";
 private static final String NUM_TOP_USERS  = 
"application.dailyJobReport.numTopUsers";
 private static final String JOB_OVERTIME_LIMIT_HOUR  = 
"application.dailyJobReport.jobOvertimeLimitInHour";
 
 public static final String SERVICE_PATH = "application.dailyJobReport";
-public static final String APP_TYPE = "MR_HISTORY_JOB_APP";
+protected static final String APP_TYPE = "MR_HISTORY_JOB_APP";
 
 // alert context keys
 protected static final String NUM_TOP_USERS_KEY = "numTopUsers";
@@ -64,6 +65,12 @@ public class MRHistoryJobDailyReporter extends 
AbstractScheduledService {
 protected static final String FINISHED_JOB_USERS_KEY = "finishedJobUsers";
 protected static final String EAGLE_JOB_LINK_KEY = "eagleJobLink";
 
+// queries
+private static final String STATUS_QUERY = "%s[@site=\"%s\" and 
@endTime<=%s]<@currentState>{count}.{count desc}";
+private static final String FAILED_JOBS_QUERY = "%s[@site=\"%s\" and 
@currentState=\"FAILED\" and @endTime<=%s]<@user>{count}.{count desc}";
+private static final String SUCCEEDED_JOB_QUERY = "%s[@site=\"%s\" and 
@currentState=\"SUCCEEDED\" and @durationTime>%s and 
@endTime<=%s]<@user>{count}.{count desc}";
+private static final String FINISHED_JOB_QUERY = "%s[@site=\"%s\" and 
@endTime<=%s]<@user>{count}.{count desc}";
+
 private Config config;
 private IEagleServiceClient client;
 private ApplicationEntityService applicationResource;
@@ -77,13 +84,13 @@ public class MRHistoryJobDailyReporter extends 
AbstractScheduledService {
 private int jobOvertimeLimit = 6;
 
 // scheduler
-private int initialDelayMin = 10;
+private int initialDelayMin = 5;
 private int periodInMin = 60;
 private TimeZone timeZone;
 
 @Inject
 public MRHistoryJobDailyReporter(Config config, ApplicationEntityService 
applicationEntityService) {
-this.timeZone = TimeZone.getTimeZone(config.getString(TIMEZONE_PATH));
+this.timeZone =

incubator-eagle git commit: [MINOR] add stream data source config for mr history job

2016-12-15 Thread jinhuwu 
Repository: incubator-eagle
Updated Branches:
  refs/heads/master e24de5c7e -> 229d7b907


[MINOR] add stream data source config for mr history job

Author: wujinhu 

Closes #748 from wujinhu/EAGLE-796.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/229d7b90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/229d7b90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/229d7b90

Branch: refs/heads/master
Commit: 229d7b9073a430de7dbbb12d39ddc330646eb458
Parents: e24de5c
Author: wujinhu 
Authored: Fri Dec 16 14:55:47 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 16 14:55:47 2016 +0800

--
 .../apache/eagle/jpm/mr/history/storm/JobHistorySpout.java  | 9 ++---
 ...eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml | 9 -
 2 files changed, 14 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/229d7b90/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
index 0cd30ae..d7daa5e 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
@@ -140,7 +140,12 @@ public class JobHistorySpout extends BaseRichSpout {
 JobHistoryZKStateManager.instance().init(appConfig.getZkStateConfig());
 JobHistoryZKStateManager.instance().ensureJobPartition(partitionId, 
numTotalPartitions);
 interceptor.setSpoutOutputCollector(collector);
-
+if (streamPublishers != null) {
+for (StreamPublisher streamPublisher : streamPublishers) {
+streamPublisher.setCollector(this.interceptor);
+
StreamPublisherManager.getInstance().addStreamPublisher(streamPublisher);
+}
+}
 try {
 jhfLCM = new JobHistoryDAOImpl(jobHistoryEndpointConfig);
 driver = new JHFCrawlerDriverImpl(
@@ -189,8 +194,6 @@ public class JobHistorySpout extends BaseRichSpout {
 if (streamPublishers != null) {
 for (StreamPublisher streamPublisher : streamPublishers) {
 declarer.declareStream(streamPublisher.stormStreamId(), new 
Fields("f1", "message"));
-streamPublisher.setCollector(this.interceptor);
-
StreamPublisherManager.getInstance().addStreamPublisher(streamPublisher);
 }
 } else {
 declarer.declare(new Fields("f1", "message"));

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/229d7b90/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index ccf3c6b..1ff9b85 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -96,6 +96,13 @@
 true
 
 
+dataSourceConfig.zkConnection
+Kafka Zookeeper Quorum
+localhost:2181
+kafka zookeeper connection
+true
+
+
 dataSinkConfig.serializerClass
 Serializer Class For Kafka Message Value
 kafka.serializer.StringEncoder
@@ -221,7 +228,7 @@
 string
 
 
-queue
+hostname
 string

incubator-eagle git commit: [MINOR] fix job & task attempt stream publisher bug in storm env

2016-12-15 Thread jinhuwu 
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8fe968cb7 -> e24de5c7e


[MINOR] fix job & task attempt stream publisher bug in storm env

Author: wujinhu 

Closes #747 from wujinhu/EAGLE-796.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e24de5c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e24de5c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e24de5c7

Branch: refs/heads/master
Commit: e24de5c7e85f7dba7eb5dfc4aee56b421c0e8ec4
Parents: 8fe968c
Author: wujinhu 
Authored: Fri Dec 16 11:23:40 2016 +0800
Committer: wujinhu 
Committed: Fri Dec 16 11:23:40 2016 +0800

--
 .../jpm/mr/history/MRHistoryJobApplication.java | 15 +---
 .../crawler/DefaultJHFInputStreamCallback.java  |  6 ++---
 .../jpm/mr/history/parser/JHFParserFactory.java |  3 +--
 .../JobEntityCreationEagleServiceListener.java  |  8 +++
 .../history/publisher/JobStreamPublisher.java   |  4 ++--
 .../mr/history/publisher/StreamPublisher.java   | 13 +--
 .../publisher/TaskAttemptStreamPublisher.java   |  4 ++--
 .../jpm/mr/history/storm/JobHistorySpout.java   | 24 
 history.MRHistoryJobApplicationProvider.xml | 15 
 9 files changed, 53 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e24de5c7/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
index e2695a3..4d02e91 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java
@@ -24,6 +24,7 @@ import 
org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilter;
 import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilterBuilder;
 import 
org.apache.eagle.jpm.mr.history.crawler.JobHistorySpoutCollectorInterceptor;
 import org.apache.eagle.jpm.mr.history.publisher.JobStreamPublisher;
+import org.apache.eagle.jpm.mr.history.publisher.StreamPublisher;
 import org.apache.eagle.jpm.mr.history.publisher.StreamPublisherManager;
 import org.apache.eagle.jpm.mr.history.publisher.TaskAttemptStreamPublisher;
 import org.apache.eagle.jpm.mr.history.storm.JobHistorySpout;
@@ -67,8 +68,7 @@ public class MRHistoryJobApplication extends StormApplication 
{
 TopologyBuilder topologyBuilder = new TopologyBuilder();
 String spoutName = "mrHistoryJobSpout";
 int tasks = jhfAppConf.getInt("stormConfig.mrHistoryJobSpoutTasks");
-JobHistorySpoutCollectorInterceptor collectorInterceptor = new 
JobHistorySpoutCollectorInterceptor();
-JobHistorySpout jobHistorySpout = new JobHistorySpout(filter, 
appConfig, collectorInterceptor);
+JobHistorySpout jobHistorySpout = new JobHistorySpout(filter, 
appConfig);
 topologyBuilder.setSpout(
 spoutName,
 jobHistorySpout,
@@ -89,14 +89,11 @@ public class MRHistoryJobApplication extends 
StormApplication {
 String spoutToTaskAttemptSinkName = spoutName + "_to_" + 
taskAttemptSinkBoltName;
 taskAttemptKafkaBoltDeclarer.shuffleGrouping(spoutName, 
spoutToTaskAttemptSinkName);
 
-List streams = new ArrayList<>();
-streams.add(spoutToJobSinkName);
-streams.add(spoutToTaskAttemptSinkName);
-jobHistorySpout.setStreams(streams);
+List streamPublishers = new ArrayList<>();
+streamPublishers.add(new JobStreamPublisher(spoutToJobSinkName));
+streamPublishers.add(new 
TaskAttemptStreamPublisher(spoutToTaskAttemptSinkName));
+jobHistorySpout.setStreamPublishers(streamPublishers);
 
-//4, add stream publisher
-StreamPublisherManager.getInstance().addStreamPublisher(new 
JobStreamPublisher(spoutToJobSinkName, collectorInterceptor));
-StreamPublisherManager.getInstance().addStreamPublisher(new 
TaskAttemptStreamPublisher(spoutToTaskAttemptSinkName, collectorInterceptor));
 return topologyBuilder.createTopology();
 }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e24de5c7/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java
--
diff --git

incubator-eagle git commit: [EAGLE-834] Improve jpm report template by fixing font-size and notes

2016-12-15 Thread hao 
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 3082b5e62 -> bb71d37df


[EAGLE-834] Improve jpm report template by fixing font-size and notes

* Improve jpm report template by fixing font-size and notes

http://issues.apache.org/jira/browse/EAGLE-834

Author: Hao Chen 

Closes #746 from haoch/ImproveJPMReport.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/bb71d37d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/bb71d37d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/bb71d37d

Branch: refs/heads/master
Commit: bb71d37df856fa7738af9d654afc3a772a0ef8f8
Parents: 3082b5e
Author: Hao Chen 
Authored: Thu Dec 15 19:36:55 2016 +0800
Committer: Hao Chen 
Committed: Thu Dec 15 19:36:55 2016 +0800

--
 .../src/main/resources/JobReportTemplate.vm | 263 ++-
 .../history/MRHistoryJobDailyReporterTest.java  |   6 +-
 2 files changed, 88 insertions(+), 181 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/bb71d37d/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
--
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
index 424306f..c3afa58 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
@@ -76,7 +76,7 @@
 }
 
 h3 {
-font-size: 16px !important;
+font-size: 14px !important;
 }
 
 .container {
@@ -119,7 +119,7 @@
bgcolor="#fff">
 
 
 $alert["alertTitle"]
 
@@ -131,17 +131,19 @@
 
 
-
-Job Summary
-
+Time: 
$alert["reportRange"]
+
 
 
 
-Time: 
$alert["reportRange"]
+
+Summary
+
 
 
 
@@ -164,7 +166,7 @@
 valign="top">
 Number of Jobs
 
-
 Ratio
 
@@ -173,13 +175,19 @@
 
 
-$item.status
+
#set($statusColor = "#337ab7")
+
#if($item.status == "SUCCEEDED")
+
#set($statusColor = "green")
+
#elseif($item.status == "FAILED" || $item.status == "KILLED")
+
#set($statusColor = "red")
+#end
+$item.status
 
 
 $item.numOfJobs
 
-
 #set 
($ratio_percentage = $item.ratio * 100)
 
$ratio_percentage %
@@ -194,12 +202,14 @@
 
 
 
-
 ##Top $alert["numTopUsers"] Users 
(Order by Number of Failed Jobs)
-