Repository: incubator-eagle Updated Branches: refs/heads/master ce8e1c508 -> 4874433a5
[MINOR] add jobConf.additional.info in spark history job app A quick fixed bug Author: Zhao, Qingwen <qingwz...@apache.org> Closes #662 from qingwen220/minor. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4874433a Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4874433a Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4874433a Branch: refs/heads/master Commit: 4874433a59c59e599b73a1235819db92e3248ce1 Parents: ce8e1c5 Author: Zhao, Qingwen <qingwz...@apache.org> Authored: Thu Nov 17 13:18:55 2016 +0800 Committer: Zhao, Qingwen <qingwz...@apache.org> Committed: Thu Nov 17 13:18:55 2016 +0800 ---------------------------------------------------------------------- .../eagle/jpm/spark/history/crawl/JHFSparkEventReader.java | 7 ++++++- ...che.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml | 6 ++++++ .../src/main/resources/application.conf | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java index a4ac77e..82e8a41 100644 --- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java +++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java @@ -115,10 +115,15 @@ public class JHFSparkEventReader { app.setConfig(new JobConfig()); JSONObject sparkProps = (JSONObject) event.get("Spark Properties"); - String[] additionalJobConf = conf.getString("basic.jobConf.additional.info").split(",\\s*"); String[] props = {"spark.yarn.app.id", "spark.executor.memory", "spark.driver.host", "spark.driver.port", "spark.driver.memory", "spark.scheduler.pool", "spark.executor.cores", "spark.yarn.am.memory", "spark.yarn.am.cores", "spark.yarn.executor.memoryOverhead", "spark.yarn.driver.memoryOverhead", "spark.yarn.am.memoryOverhead", "spark.master"}; + + String[] additionalJobConf = null; + if (conf.hasPath("spark.jobConf.additional.info")) { + additionalJobConf = conf.getString("spark.jobConf.additional.info").split(",\\s*"); + } + String[] jobConf = (String[]) ArrayUtils.addAll(additionalJobConf, props); for (String prop : jobConf) { if (sparkProps.containsKey(prop)) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml index c68d4e8..17a3a4a 100644 --- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml +++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml @@ -90,6 +90,12 @@ </property> <property> + <name>spark.jobConf.additional.info</name> + <displayName>additional spark job config info</displayName> + <description>additional job config list</description> + <value></value> + </property> + <property> <name>spark.defaultVal.spark.executor.memory</name> <displayName>spark.executor.memory</displayName> <value>1g</value> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf index a51abc9..2839915 100644 --- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf +++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf @@ -57,6 +57,7 @@ numOfParserBoltTasks: 4 }, "spark":{ + jobConf.additional.info: "", "defaultVal": { spark.executor.memory:"1g", spark.driver.memory: "1g",
