Repository: oozie Updated Branches: refs/heads/branch-4.2 46402ac45 -> ae9d5afc0
OOZIE-2350 Package changes for release (shwethags) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ae9d5afc Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ae9d5afc Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ae9d5afc Branch: refs/heads/branch-4.2 Commit: ae9d5afc02b5c8e6c9969fb0bc106a958c37fa6e Parents: 46402ac Author: Shwetha GS <[email protected]> Authored: Thu Sep 3 10:46:24 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Thu Sep 3 10:46:24 2015 +0530 ---------------------------------------------------------------------- pom.xml | 15 +-- release-log.txt | 1 + .../SparkMain.java | 106 ------------------- .../apache/oozie/action/hadoop/SparkMain.java | 106 +++++++++++++++++++ 4 files changed, 108 insertions(+), 120 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/ae9d5afc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ee26366..12d9977 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.apache</groupId> <artifactId>apache</artifactId> - <version>16</version> + <version>17</version> </parent> <modelVersion>4.0.0</modelVersion> @@ -119,19 +119,6 @@ <module>zookeeper-security-tests</module> </modules> - <distributionManagement> - <repository> - <id>apache.staging.https</id> - <name>Apache Release Distribution Repository</name> - <url>https://repository.apache.org/service/local/staging/deploy/maven2</url> - </repository> - <snapshotRepository> - <id>apache.snapshots.https</id> - <name>${distMgmtSnapshotsName}</name> - <url>${distMgmtSnapshotsUrl}</url> - </snapshotRepository> - </distributionManagement> - <repositories> <repository> <id>central</id> http://git-wip-us.apache.org/repos/asf/oozie/blob/ae9d5afc/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 737c768..a164565 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release +OOZIE-2350 Package changes for release (shwethags) OOZIE-2240 add configuration to disable email attachment support (egashira via shwethags) OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags) OOZIE-1993 Rerun fails during join in certain condition (shwethags) http://git-wip-us.apache.org/repos/asf/oozie/blob/ae9d5afc/sharelib/spark/src/main/java/org.apache.oozie.action.hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org.apache.oozie.action.hadoop/SparkMain.java b/sharelib/spark/src/main/java/org.apache.oozie.action.hadoop/SparkMain.java deleted file mode 100644 index b18a0b9..0000000 --- a/sharelib/spark/src/main/java/org.apache.oozie.action.hadoop/SparkMain.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.spark.deploy.SparkSubmit; - -import java.util.ArrayList; -import java.util.List; - -public class SparkMain extends LauncherMain { - private static final String MASTER_OPTION = "--master"; - private static final String MODE_OPTION = "--deploy-mode"; - private static final String JOB_NAME_OPTION = "--name"; - private static final String CLASS_NAME_OPTION = "--class"; - private static final String VERBOSE_OPTION = "--verbose"; - private static final String DELIM = " "; - - - public static void main(String[] args) throws Exception { - run(SparkMain.class, args); - } - - @Override - protected void run(String[] args) throws Exception { - Configuration actionConf = loadActionConf(); - setYarnTag(actionConf); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); - - List<String> sparkArgs = new ArrayList<String>(); - - sparkArgs.add(MASTER_OPTION); - sparkArgs.add(actionConf.get(SparkActionExecutor.SPARK_MASTER)); - - String sparkDeployMode = actionConf.get(SparkActionExecutor.SPARK_MODE); - if (sparkDeployMode != null) { - sparkArgs.add(MODE_OPTION); - sparkArgs.add(sparkDeployMode); - } - - sparkArgs.add(JOB_NAME_OPTION); - sparkArgs.add(actionConf.get(SparkActionExecutor.SPARK_JOB_NAME)); - - String className = actionConf.get(SparkActionExecutor.SPARK_CLASS); - if (className != null) { - sparkArgs.add(CLASS_NAME_OPTION); - sparkArgs.add(className); - } - - String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS); - if (StringUtils.isNotEmpty(sparkOpts)) { - String[] sparkOptions = sparkOpts.split(DELIM); - for (String opt : sparkOptions) { - sparkArgs.add(opt); - } - } - - if (!sparkArgs.contains(VERBOSE_OPTION)) { - sparkArgs.add(VERBOSE_OPTION); - } - - String jarPath = actionConf.get(SparkActionExecutor.SPARK_JAR); - sparkArgs.add(jarPath); - - for (String arg : args) { - sparkArgs.add(arg); - } - - System.out.println("Spark Action Main class : " + SparkSubmit.class.getName()); - System.out.println(); - System.out.println("Oozie Spark action configuration"); - System.out.println("================================================================="); - System.out.println(); - for (String arg : sparkArgs) { - System.out.println(" " + arg); - } - System.out.println(); - runSpark(sparkArgs.toArray(new String[sparkArgs.size()])); - } - - private void runSpark(String[] args) throws Exception { - System.out.println("================================================================="); - System.out.println(); - System.out.println(">>> Invoking Spark class now >>>"); - System.out.println(); - System.out.flush(); - SparkSubmit.main(args); - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/ae9d5afc/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java new file mode 100644 index 0000000..b18a0b9 --- /dev/null +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.deploy.SparkSubmit; + +import java.util.ArrayList; +import java.util.List; + +public class SparkMain extends LauncherMain { + private static final String MASTER_OPTION = "--master"; + private static final String MODE_OPTION = "--deploy-mode"; + private static final String JOB_NAME_OPTION = "--name"; + private static final String CLASS_NAME_OPTION = "--class"; + private static final String VERBOSE_OPTION = "--verbose"; + private static final String DELIM = " "; + + + public static void main(String[] args) throws Exception { + run(SparkMain.class, args); + } + + @Override + protected void run(String[] args) throws Exception { + Configuration actionConf = loadActionConf(); + setYarnTag(actionConf); + LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + + List<String> sparkArgs = new ArrayList<String>(); + + sparkArgs.add(MASTER_OPTION); + sparkArgs.add(actionConf.get(SparkActionExecutor.SPARK_MASTER)); + + String sparkDeployMode = actionConf.get(SparkActionExecutor.SPARK_MODE); + if (sparkDeployMode != null) { + sparkArgs.add(MODE_OPTION); + sparkArgs.add(sparkDeployMode); + } + + sparkArgs.add(JOB_NAME_OPTION); + sparkArgs.add(actionConf.get(SparkActionExecutor.SPARK_JOB_NAME)); + + String className = actionConf.get(SparkActionExecutor.SPARK_CLASS); + if (className != null) { + sparkArgs.add(CLASS_NAME_OPTION); + sparkArgs.add(className); + } + + String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS); + if (StringUtils.isNotEmpty(sparkOpts)) { + String[] sparkOptions = sparkOpts.split(DELIM); + for (String opt : sparkOptions) { + sparkArgs.add(opt); + } + } + + if (!sparkArgs.contains(VERBOSE_OPTION)) { + sparkArgs.add(VERBOSE_OPTION); + } + + String jarPath = actionConf.get(SparkActionExecutor.SPARK_JAR); + sparkArgs.add(jarPath); + + for (String arg : args) { + sparkArgs.add(arg); + } + + System.out.println("Spark Action Main class : " + SparkSubmit.class.getName()); + System.out.println(); + System.out.println("Oozie Spark action configuration"); + System.out.println("================================================================="); + System.out.println(); + for (String arg : sparkArgs) { + System.out.println(" " + arg); + } + System.out.println(); + runSpark(sparkArgs.toArray(new String[sparkArgs.size()])); + } + + private void runSpark(String[] args) throws Exception { + System.out.println("================================================================="); + System.out.println(); + System.out.println(">>> Invoking Spark class now >>>"); + System.out.println(); + System.out.flush(); + SparkSubmit.main(args); + } +}
