Repository: eagle Updated Branches: refs/heads/master c75eadd44 -> d766f6815
http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java index a987bd8..9c1a2c7 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java @@ -21,9 +21,12 @@ package org.apache.eagle.jpm.analyzer.util; import com.typesafe.config.Config; import org.apache.eagle.common.rest.RESTResponse; import org.apache.eagle.jpm.analyzer.meta.model.JobMetaEntity; +import org.apache.eagle.jpm.analyzer.meta.model.UserEmailEntity; import org.apache.eagle.jpm.util.resourcefetch.connection.InputStreamUtils; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +43,7 @@ public class Utils { OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); } - public static List<JobMetaEntity> getJobMeta(Config config, String jobDefId) { + public static List<JobMetaEntity> getJobMeta(Config config, String siteId, String jobDefId) { List<JobMetaEntity> result = new ArrayList<>(); String url = "http://"; + config.getString(Constants.HOST_PATH) @@ -48,7 +51,9 @@ public class Utils { + config.getInt(Constants.PORT_PATH) + config.getString(Constants.CONTEXT_PATH) + Constants.ANALYZER_PATH - + Constants.META_PATH + + Constants.JOB_META_ROOT_PATH + + "/" + + siteId + "/" + URLEncoder.encode(jobDefId); @@ -56,7 +61,7 @@ public class Utils { try { is = InputStreamUtils.getInputStream(url, null, org.apache.eagle.jpm.util.Constants.CompressionType.NONE); LOG.info("get job meta from {}", url); - result = (List<JobMetaEntity>)OBJ_MAPPER.readValue(is, RESTResponse.class).getData(); + result = ((RESTResponse<List<JobMetaEntity>>)OBJ_MAPPER.readValue(is, new TypeReference<RESTResponse<List<JobMetaEntity>>>(){})).getData(); } catch (Exception e) { LOG.warn("failed to get job meta from {}", url, e); } finally { @@ -65,6 +70,33 @@ public class Utils { } } + public static List<UserEmailEntity> getUserMail(Config config, String siteId, String userId) { + List<UserEmailEntity> result = new ArrayList<>(); + String url = "http://"; + + config.getString(Constants.HOST_PATH) + + ":" + + config.getInt(Constants.PORT_PATH) + + config.getString(Constants.CONTEXT_PATH) + + Constants.ANALYZER_PATH + + Constants.USER_META_ROOT_PATH + + "/" + + siteId + + "/" + + URLEncoder.encode(userId); + + InputStream is = null; + try { + is = InputStreamUtils.getInputStream(url, null, org.apache.eagle.jpm.util.Constants.CompressionType.NONE); + LOG.info("get user meta from {}", url); + result = ((RESTResponse<List<UserEmailEntity>>)OBJ_MAPPER.readValue(is, new TypeReference<RESTResponse<List<UserEmailEntity>>>(){})).getData(); + } catch (Exception e) { + LOG.warn("failed to get user meta from {}", url, e); + } finally { + org.apache.eagle.jpm.util.Utils.closeInputStream(is); + return result; + } + } + public static <K, V extends Comparable<? super V>> List<Map.Entry<K, V>> sortByValue(Map<K, V> map) { List<Map.Entry<K, V>> list = new LinkedList<>(map.entrySet()); Collections.sort(list, (e1, e2) -> e1.getValue().compareTo(e2.getValue())); http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm index 996adba..0486a33 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm @@ -14,11 +14,27 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> -<html xmlns="http://www.w3.org/1999/xhtml";> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";> + + #set ( $alert = $alertList[0] ) + +## Generate Alert Color +#set($alertColor = "#337ab7") +#if($alert["alertSeverity"] == "WARNING") + #set($alertColor = "#FF9F00") +#elseif($alert["alertSeverity"] == "CRITICAL") + #set($alertColor = "#d43f3a") +#elseif ($alert["alertSeverity"] == "INFO") + #set($alertColor = "#68B90F") +#end + +<html xmlns="http://www.w3.org/1999/xhtml"; + style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> <head> - <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> <meta name="viewport" content="width=device-width"/> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <title>[$alert["alertSeverity"] $alert["alertSubject"]</title> <style type="text/css"> img { max-width: 100%; @@ -62,11 +78,11 @@ } h1 { - font-size: 22px !important; + font-size: 20px !important; } h2 { - font-size: 18px !important; + font-size: 16px !important; } h3 { @@ -83,7 +99,7 @@ } .content-wrap { - padding: 10px !important; + padding: 0px !important; } .invoice { @@ -92,44 +108,159 @@ } </style> </head> -<body> - #set ( $elem = $alertList[0] ) - -<p><b>Basic Information: </b></p> - -<ul> - <li>Site: ${elem["basic"].get("site")}</li> - <li>Job Name: ${elem["basic"].get("name")}</li> - <li>User: ${elem["basic"].get("user")}</li> - <li>Job Status: ${elem["basic"].get("status")}</li> - <li>Start Time: ${elem["basic"].get("start")}</li> - <li>End Time: ${elem["basic"].get("end")}</li> - <li>Duration Time: ${elem["basic"].get("duration")}</li> - <li>Progress: ${elem["basic"].get("progress")}</li> - <li>Job Detail: <a href=${elem["basic"].get("detail")}>${elem["basic"].get("detail")}</a></li> -</ul> - -<p><b>Analyzer Results: </b></p> - -#foreach($evaluator in ${elem["extend"].keySet()}) -<table class="body-wrap" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6" border="1"> - <caption><b>Analysis By $evaluator</b></caption> - <tr> - <th style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" width="100"><b>type</b></th> - <th style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" width="250"><b>message</b></th> - <th style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" width="250"><b>optimizer setting</b></th> - <th style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" width="100"><b>level</b></th> + +<body itemscope itemtype="http://schema.org/EmailMessage"; + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; margin: 0;"> + +<table class="body-wrap" + style="..." + bgcolor="#f6f6f6" align="center"> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"></td> + <td class="container" width="600" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;" + valign="top"> + <div class="content" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"> + <table class="main" width="100%" cellpadding="0" cellspacing="0" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;"> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td class="alert alert-warning" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: $alertColor; margin: 0; padding: 20px;" + align="center" bgcolor="$alertColor" valign="top"> + <strong >$alert["alertSeverity"]: </strong> + $alert["alertSubject"] + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td class="content-wrap" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + <br> + <table class="invoice" cellpadding="1" cellspacing="1" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; margin: 0; border: 1px solid #e9e9e9" bgcolor="#f6f6f6"> + <caption><b>Basic Information</b></caption> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Severity: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> $alert["alertSeverity"] + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Site: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("site")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Name: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("name")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + User: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("user")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Status: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("status")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Progress: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("progress")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Start: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("start")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + End: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("end")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Duration: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> ${alert["basic"].get("duration")} + </td> + </tr> + <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> + Detail: + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"> <a href=${alert["basic"].get("detail")}>${alert["basic"].get("detail")}</a> + </td> + </tr> + </table> + <br> + <table class="invoice" cellpadding="1" cellspacing="1" + style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; margin: 0; border: 1px solid #e9e9e9; border-collapse: collapse" bgcolor="#f6f6f6"> + <caption><b>Analysis Results</b></caption> + <tr> + <th style="border: 1px solid #e9e9e9" + valign="top"><b>level</b></th> + <th style="border: 1px solid #e9e9e9" + valign="top" align="center"><b>message</b></th> + </tr> + #foreach($evaluator in ${alert["extend"].keySet()}) + #foreach($result in ${alert["extend"].get($evaluator)}) + <tr style="..." + valign="top"> + <td style="border: 1px solid #e9e9e9">${result.resultLevel}</td> + <td style="border: 1px solid #e9e9e9">${result.message}</td> + </tr> + #end + #end + </table> + + </td> + </tr> + </table> + </div> + </td> + <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" + valign="top"></td> </tr> - #foreach($result in ${elem["extend"].get($evaluator)}) - <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> - <td style="...">${result.ruleType}</td> - <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">${result.message}</td> - <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">${result.settingList}</td> - <td style="...">${result.resultLevel}</td> - </tr> - #end </table> -#end - </body> -</html> \ No newline at end of file +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-analyzer/src/main/resources/createTable.sql ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/createTable.sql b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/createTable.sql index 78fd02a..7d725d2 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/createTable.sql +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/createTable.sql @@ -16,26 +16,23 @@ -- * -- */ -CREATE TABLE IF NOT EXISTS jobs ( - jobDefId VARCHAR(50) NOT NULL, - configuration MEDIUMTEXT NOT NULL, +CREATE TABLE IF NOT EXISTS analysis_jobs ( + uuid varchar(50) PRIMARY KEY, + jobDefId varchar(100) NOT NULL, + siteId varchar(50) NOT NULL, + configuration mediumtext NOT NULL, + evaluators mediumtext NOT NULL, createdtime bigint(20) DEFAULT NULL, modifiedtime bigint(20) DEFAULT NULL, - PRIMARY KEY (jobDefId) + UNIQUE (siteId, jobDefId) ); -CREATE TABLE IF NOT EXISTS job_evaluators ( - jobDefId VARCHAR(50) NOT NULL, - evaluator VARCHAR(100) NOT NULL, - createdtime bigint(20) DEFAULT NULL, - modifiedtime bigint(20) DEFAULT NULL, - PRIMARY KEY (jobDefId, evaluator) -); - -CREATE TABLE IF NOT EXISTS job_publishments ( - userId VARCHAR(100) PRIMARY KEY, +CREATE TABLE IF NOT EXISTS analysis_email ( + uuid varchar(50) PRIMARY KEY, + userId varchar(100) NOT NULL, + siteId varchar(50) NOT NULL, mailAddress mediumtext NOT NULL, createdtime bigint(20) DEFAULT NULL, modifiedtime bigint(20) DEFAULT NULL, - PRIMARY KEY (userId) + UNIQUE (siteId, userId) ); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationProvider.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationProvider.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationProvider.java index 8751e73..30c63a8 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationProvider.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationProvider.java @@ -19,11 +19,14 @@ package org.apache.eagle.jpm.mr.history; import com.codahale.metrics.health.HealthCheck; import com.google.common.util.concurrent.Service; import com.typesafe.config.Config; -import io.dropwizard.lifecycle.Managed; import org.apache.eagle.app.service.ApplicationListener; import org.apache.eagle.app.spi.AbstractApplicationProvider; +import org.apache.eagle.jpm.analyzer.meta.MetaManagementService; +import org.apache.eagle.jpm.analyzer.meta.impl.MetaManagementServiceJDBCImpl; +import org.apache.eagle.jpm.analyzer.meta.impl.MetaManagementServiceMemoryImpl; +import org.apache.eagle.metadata.service.memory.MemoryMetadataStore; +import org.apache.eagle.metadata.store.jdbc.JDBCMetadataStore; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -52,4 +55,10 @@ public class MRHistoryJobApplicationProvider extends AbstractApplicationProvider return Optional.empty(); } } + + @Override + protected void onRegister() { + bind(MemoryMetadataStore.class, MetaManagementService.class, MetaManagementServiceMemoryImpl.class); + bind(JDBCMetadataStore.class, MetaManagementService.class, MetaManagementServiceJDBCImpl.class); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java index 2c9dc8f..2e56632 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java @@ -38,7 +38,7 @@ import java.util.regex.Pattern; public class JHFCrawlerDriverImpl implements JHFCrawlerDriver { private static final Logger LOG = LoggerFactory.getLogger(JHFCrawlerDriverImpl.class); - private static final int SLEEP_SECONDS_WHILE_QUEUE_IS_EMPTY = 10; + private static final int SLEEP_SECONDS_WHILE_QUEUE_IS_EMPTY = 120; private static final String FORMAT_JOB_PROCESS_DATE = "%4d%02d%02d"; private static final Pattern PATTERN_JOB_PROCESS_DATE = Pattern.compile("([0-9]{4})([0-9]{2})([0-9]{2})"); http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java index 724c62c..b174a2f 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -74,6 +74,7 @@ public class JobSuggestionListener implements HistoryJobEntityCreationListener { info.setFailedMaps(jobExecutionAPIEntity.getNumFailedMaps()); info.setTotalMaps(jobExecutionAPIEntity.getNumTotalMaps()); info.setTotalReduces(jobExecutionAPIEntity.getNumTotalReduces()); + info.setProgress(100); } } @@ -89,14 +90,6 @@ public class JobSuggestionListener implements HistoryJobEntityCreationListener { @Override public void flush() throws Exception { - if (info.getTotalCounters() == null) { - LOG.warn("Total counters of Job {} is null", info.getJobId()); - return; - } - if (info.getMapCounters() == null && info.getReduceCounters() == null) { - LOG.warn("Map/Reduce task counters of Job {} are null", info.getJobId()); - return; - } analyzer.analyze(info); } } http://git-wip-us.apache.org/repos/asf/eagle/blob/d766f681/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java index d7daa5e..11deefe 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java @@ -175,7 +175,7 @@ public class JobHistorySpout extends BaseRichSpout { } } finally { try { - Thread.sleep(1000); + Thread.sleep(5000); } catch (Exception e) { // ignored }
