Repository: oozie Updated Branches: refs/heads/master 7779f3184 -> fd7708949
OOZIE-2563 Pass spark-defaults.conf to spark action (satishsaley via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/fd770894 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/fd770894 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/fd770894 Branch: refs/heads/master Commit: fd7708949e0ae5eb0b65a9f6ff5e4af09a6dde57 Parents: 7779f31 Author: Rohini Palaniswamy <[email protected]> Authored: Wed Jun 8 15:29:37 2016 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Wed Jun 8 15:29:37 2016 -0700 ---------------------------------------------------------------------- release-log.txt | 1 + .../apache/oozie/action/hadoop/SparkMain.java | 29 ++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/fd770894/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 2f69344..5fd8ef8 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2563 Pass spark-defaults.conf to spark action (satishsaley via rohini) OOZIE-2556 TestAbandonedCoordChecker.testCatchupJob is flaky (puru) OOZIE-2522 There can be multiple coord submit from bundle in case of ZK glitch (puru) OOZIE-2553 Cred tag is required for all actions in the workflow even if an action does not require it (me.venkatr via rohini) http://git-wip-us.apache.org/repos/asf/oozie/blob/fd770894/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 0e6e271..13c1075 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -50,6 +50,8 @@ public class SparkMain extends LauncherMain { private static final Pattern[] PYSPARK_DEP_FILE_PATTERN = { Pattern.compile("py4\\S*src.zip"), Pattern.compile("pyspark.zip") }; + private static final Pattern SPARK_DEFAULTS_FILE_PATTERN = Pattern.compile("spark-defaults.conf"); + private String sparkJars = null; private String sparkClasspath = null; @@ -198,6 +200,11 @@ public class SparkMain extends LauncherMain { sparkArgs.add("--conf"); sparkArgs.add(HBASE_SECURITY_TOKEN + "=false"); } + File defaultConfFile = getMatchingFile(SPARK_DEFAULTS_FILE_PATTERN); + if (defaultConfFile != null) { + sparkArgs.add("--properties-file"); + sparkArgs.add(SPARK_DEFAULTS_FILE_PATTERN.toString()); + } if (!sparkArgs.contains(VERBOSE_OPTION)) { sparkArgs.add(VERBOSE_OPTION); } @@ -242,7 +249,7 @@ public class SparkMain extends LauncherMain { } for(Pattern fileNamePattern : PYSPARK_DEP_FILE_PATTERN) { - File file = getMatchingFile(fileNamePattern); + File file = getMatchingPyFile(fileNamePattern); File destination = new File(pythonLibDir, file.getName()); FileUtils.copyFile(file, destination); System.out.println("Copied " + file + " to " + destination.getAbsolutePath()); @@ -256,6 +263,23 @@ public class SparkMain extends LauncherMain { * @return the file if there is one * @throws OozieActionConfiguratorException if there is are no files matching the pattern */ + private File getMatchingPyFile(Pattern fileNamePattern) throws OozieActionConfiguratorException { + File f = getMatchingFile(fileNamePattern); + if (f != null) { + return f; + } + throw new OozieActionConfiguratorException("Missing py4j and/or pyspark zip files. Please add them to " + + "the lib folder or to the Spark sharelib."); + } + + /** + * Searches for a file in the current directory that matches the given + * pattern. If there are multiple files matching the pattern returns one of + * them. + * + * @param fileNamePattern the pattern to look for + * @return the file if there is one else it returns null + */ private File getMatchingFile(Pattern fileNamePattern) throws OozieActionConfiguratorException { File localDir = new File("."); for(String fileName : localDir.list()){ @@ -263,8 +287,7 @@ public class SparkMain extends LauncherMain { return new File(fileName); } } - throw new OozieActionConfiguratorException("Missing py4j and/or pyspark zip files. Please add them to " + - "the lib folder or to the Spark sharelib."); + return null; } private void runSpark(String[] args) throws Exception {
