Repository: incubator-eagle Updated Branches: refs/heads/master 6ec928941 -> ab3ed13e1
[EAGLE-728] Application can't load application.properties when running local mode EAGLE-728 Application can't load application.properties when running local mode - Use hard code, and overwrite it if user passes this configuration form web ui. https://issues.apache.org/jira/browse/EAGLE-728 Author: chitin <chitin1...@gmail.com> Closes #612 from chitin/EAGLE-728-2. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ab3ed13e Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ab3ed13e Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ab3ed13e Branch: refs/heads/master Commit: ab3ed13e18c85fa3fd64a08a070aaf24391aeb1e Parents: 6ec9289 Author: chitin <chitin1...@gmail.com> Authored: Wed Nov 9 10:47:06 2016 +0800 Committer: wujinhu <wujinhu...@126.com> Committed: Wed Nov 9 10:47:06 2016 +0800 ---------------------------------------------------------------------- .../jpm/mr/history/parser/JHFEventReaderBase.java | 2 +- .../eagle/jpm/mr/running/MRRunningJobApplication.java | 3 ++- .../eagle/jpm/mr/running/parser/MRJobParser.java | 8 ++++++-- .../jpm/mr/running/storm/MRRunningJobParseBolt.java | 8 ++++++-- .../jpm/spark/history/crawl/JHFSparkEventReader.java | 2 +- .../java/org/apache/eagle/jpm/util/Constants.java | 4 ++++ .../apache/eagle/jpm/util/JobNameNormalization.java | 14 ++++---------- 7 files changed, 24 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index e48370d..80cdb1c 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -200,7 +200,7 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl } if (jobDefId == null) { - this.jobDefId = JobNameNormalization.getInstance().normalize(jobName); + this.jobDefId = JobNameNormalization.getInstance(this.appConfig.getConfig()).normalize(jobName); } else { LOG.debug("Got JobDefId from job configuration for " + id + ": " + jobDefId); this.jobDefId = jobDefId; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java index bfdde13..e5c7c87 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java @@ -66,7 +66,8 @@ public class MRRunningJobApplication extends StormApplication { mrRunningJobConfig.getEagleServiceConfig(), mrRunningJobConfig.getEndpointConfig(), mrRunningJobConfig.getZkStateConfig(), - confKeyKeys), + confKeyKeys, + config), tasks).setNumTasks(tasks).fieldsGrouping(spoutName, new Fields("appId")); return topologyBuilder.createTopology(); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java index 945e8ce..7b90dfc 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java @@ -18,6 +18,7 @@ package org.apache.eagle.jpm.mr.running.parser; +import com.typesafe.config.Config; import org.apache.eagle.jpm.mr.running.MRRunningJobConfig; import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager; import org.apache.eagle.jpm.mr.runningentity.JobConfig; @@ -79,6 +80,7 @@ public class MRJobParser implements Runnable { private static final int TOP_BOTTOM_TASKS_BY_ELAPSED_TIME = 10; private static final int FLUSH_TASKS_EVERY_TIME = 5; private static final int MAX_TASKS_PERMIT = 5000; + private Config config; static { OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); @@ -88,7 +90,8 @@ public class MRJobParser implements Runnable { MRRunningJobConfig.EagleServiceConfig eagleServiceConfig, AppInfo app, Map<String, JobExecutionAPIEntity> mrJobMap, MRRunningJobManager runningJobManager, ResourceFetcher rmResourceFetcher, - List<String> configKeys) { + List<String> configKeys, + Config config) { this.app = app; this.mrJobEntityMap = new HashMap<>(); this.mrJobEntityMap = mrJobMap; @@ -107,6 +110,7 @@ public class MRJobParser implements Runnable { this.rmResourceFetcher = rmResourceFetcher; this.finishedTaskIds = new HashSet<>(); this.configKeys = configKeys; + this.config = config; } public void setAppInfo(AppInfo app) { @@ -195,7 +199,7 @@ public class MRJobParser implements Runnable { jobExecutionAPIEntity.setTags(new HashMap<>(commonTags)); jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_ID.toString(), id); jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), mrJob.getName()); - String jobDefId = JobNameNormalization.getInstance().normalize(mrJob.getName()); + String jobDefId = JobNameNormalization.getInstance(this.config).normalize(mrJob.getName()); jobExecutionAPIEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId); if (mrJobConfigs.get(id) != null) { JobConfig jobConfig = mrJobConfigs.get(id); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java index 9ebc1a7..e2767d8 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java @@ -18,6 +18,7 @@ package org.apache.eagle.jpm.mr.running.storm; +import com.typesafe.config.Config; import org.apache.eagle.jpm.mr.running.MRRunningJobConfig; import org.apache.eagle.jpm.mr.running.parser.MRJobParser; import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager; @@ -49,16 +50,19 @@ public class MRRunningJobParseBolt extends BaseRichBolt { private MRRunningJobConfig.EagleServiceConfig eagleServiceConfig; private ResourceFetcher resourceFetcher; private List<String> configKeys; + private Config config; public MRRunningJobParseBolt(MRRunningJobConfig.EagleServiceConfig eagleServiceConfig, MRRunningJobConfig.EndpointConfig endpointConfig, MRRunningJobConfig.ZKStateConfig zkStateConfig, - List<String> configKeys) { + List<String> configKeys, + Config config) { this.eagleServiceConfig = eagleServiceConfig; this.endpointConfig = endpointConfig; this.runningMRParsers = new HashMap<>(); this.zkStateConfig = zkStateConfig; this.configKeys = configKeys; + this.config = config; } @Override @@ -79,7 +83,7 @@ public class MRRunningJobParseBolt extends BaseRichBolt { MRJobParser applicationParser; if (!runningMRParsers.containsKey(appInfo.getId())) { applicationParser = new MRJobParser(endpointConfig, eagleServiceConfig, - appInfo, mrJobs, runningJobManager, this.resourceFetcher, configKeys); + appInfo, mrJobs, runningJobManager, this.resourceFetcher, configKeys, this.config); runningMRParsers.put(appInfo.getId(), applicationParser); LOG.info("create application parser for {}", appInfo.getId()); } else { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java index 82b7607..a4ac77e 100644 --- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java +++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java @@ -674,7 +674,7 @@ public class JHFSparkEventReader { if (null != assignedName) { return assignedName; } else { - return JobNameNormalization.getInstance().normalize(jobName); + return JobNameNormalization.getInstance(this.config.getConfig()).normalize(jobName); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java index 73100a9..038618f 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java @@ -187,4 +187,8 @@ public class Constants { public static final String HADOOP_HISTORY_TOTAL_METRIC_FORMAT = "hadoop.%s.history.%s"; public static final String HADOOP_HISTORY_MINUTE_METRIC_FORMAT = "hadoop.%s.history.minute.%s"; + + public static final String JOB_NAME_NORMALIZATION_RULES_KEY_DEFAULT = "^(.*)[0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}$ => " + + "$1~ ; ^(oozie:launcher):T=(.*):W=(.*):A=(.*):ID=(?:.*)$ => $1-$2-$3-$4~ ; ^(.*)([0-9]{10})$ => $1~ ; " + + "^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$ => uuid~ ; ^(.*)(?:[0-9]{8}/[0-9]{2}_[0-9]{2})$ => $1~"; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java index b3673ea..0b38c67 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java @@ -29,7 +29,6 @@ import java.util.regex.Pattern; public class JobNameNormalization { private static Logger LOG = LoggerFactory.getLogger(JobNameNormalization.class); - private static JobNameNormalization instance = new JobNameNormalization(); private static final String JOB_NAME_NORMALIZATION_RULES_KEY = "job.name.normalization.rules.key"; private static final String PARAMETERIZED_PREFIX = "\\$"; private static final String MULTIPLE_RULE_DILIMITER = ";"; @@ -59,16 +58,11 @@ public class JobNameNormalization { String target; } - private JobNameNormalization() { + private JobNameNormalization(Config conf) { try { // load normalization rules - Config conf = ConfigFactory.load(); String key = JOB_NAME_NORMALIZATION_RULES_KEY.toLowerCase(); - String value = conf.getString(key); - if (value == null) { - LOG.info("no job name normalization rules are loaded"); - return; - } + String value = conf.getString(key) != null ? conf.getString(key) : Constants.JOB_NAME_NORMALIZATION_RULES_KEY_DEFAULT; // multiple rules are concatenated with semicolon, i.e. ; String[] rules = value.split(MULTIPLE_RULE_DILIMITER); for (String rule : rules) { @@ -82,8 +76,8 @@ public class JobNameNormalization { } } - public static JobNameNormalization getInstance() { - return instance; + public static JobNameNormalization getInstance(Config config) { + return new JobNameNormalization(config); } private void addRule(String rule) {
