Addressing code review comments by rkanter Change-Id: I9ee84d850ea16b9060996d5ccfe62c9f237f2e7a
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/8c4a8565 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/8c4a8565 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/8c4a8565 Branch: refs/heads/oya Commit: 8c4a8565090ae5530f7d24c54160eac05ae5e9d0 Parents: 7060f14 Author: Peter Bacsko <[email protected]> Authored: Thu Jan 19 14:07:23 2017 +0100 Committer: Peter Bacsko <[email protected]> Committed: Thu Jan 19 14:07:29 2017 +0100 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 8 ++--- .../oozie/service/HadoopAccessorService.java | 11 +++++- .../org/apache/oozie/client/TestOozieCLI.java | 4 +-- .../service/TestHadoopAccessorService.java | 2 -- distro/src/main/bin/addtowar.sh | 2 +- docs/src/site/twiki/DG_QuickStart.twiki | 4 +-- docs/src/site/twiki/ENG_Building.twiki | 4 +-- findbugs-filter.xml | 3 ++ pom.xml | 26 -------------- .../apache/oozie/action/hadoop/Hive2Main.java | 2 -- .../oozie/action/hadoop/HdfsOperations.java | 38 ++++++++------------ .../apache/oozie/action/hadoop/LauncherAM.java | 30 +++++++--------- .../oozie/action/hadoop/MapReduceMain.java | 6 +--- .../oozie/action/hadoop/TestLauncherAM.java | 30 +++++++--------- sharelib/spark/pom.xml | 21 +---------- 15 files changed, 66 insertions(+), 125 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index eb59ff4..9114da8 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1503,17 +1503,15 @@ public class JavaActionExecutor extends ActionExecutor { context.setExternalStatus(KILLED); context.setExecutionData(KILLED, null); } catch (Exception ex) { - LOG.error("Error: ", ex); + LOG.error("Error when killing YARN application", ex); throw convertException(ex); } finally { try { FileSystem actionFs = context.getAppFileSystem(); cleanUpActionDir(actionFs, context); - if (yarnClient != null) { - yarnClient.close(); - } + Closeables.closeQuietly(yarnClient); } catch (Exception ex) { - LOG.error("Error: ", ex); + LOG.error("Error when cleaning up action dir", ex); throw convertException(ex); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 3a36588..7713196 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -532,12 +532,21 @@ public class HadoopAccessorService implements Service { } } + /** + * Return a JobClient created with the provided user/group. + * + * + * @param conf Configuration with all necessary information to create the + * JobClient. + * @return JobClient created with the provided user/group. + * @throws HadoopAccessorException if the client could not be created. + */ public JobClient createJobClient(String user, Configuration conf) throws HadoopAccessorException { return createJobClient(user, new JobConf(conf)); } /** - * Return a YarnClient created with the provided user and configuration. + * Return a YarnClient created with the provided user and configuration. The caller is responsible for closing it when done. * * @param user The username to impersonate * @param conf The conf http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index 33f602e..ceac0eb 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -134,8 +134,8 @@ public class TestOozieCLI extends DagServletTestCase { String path = getTestCaseDir() + "/" + getName() + ".properties"; Properties props = new Properties(); props.setProperty(OozieClient.USER_NAME, getTestUser()); - props.setProperty(XOozieClient.NN, "localhost:9000"); - props.setProperty(XOozieClient.RM, "localhost:9001"); + props.setProperty(XOozieClient.NN, "localhost:8020"); + props.setProperty(XOozieClient.RM, "localhost:8032"); props.setProperty("oozie.libpath", appPath); props.setProperty("mapred.output.dir", appPath); props.setProperty("a", "A"); http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java index e671b7a..1e80abe 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java @@ -20,13 +20,11 @@ package org.apache.oozie.service; import org.apache.hadoop.fs.Path; import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.security.authorize.*; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.oozie.test.XFsTestCase; import org.apache.hadoop.mapred.JobConf; http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/distro/src/main/bin/addtowar.sh ---------------------------------------------------------------------- diff --git a/distro/src/main/bin/addtowar.sh b/distro/src/main/bin/addtowar.sh index 9f8d53d..688ba6b 100644 --- a/distro/src/main/bin/addtowar.sh +++ b/distro/src/main/bin/addtowar.sh @@ -95,7 +95,7 @@ function checkOption() { #get the list of hadoop jars that will be injected based on the hadoop version function getHadoopJars() { version=$1 - if [[ "${version}" =~ 2.* ]]; then + if [[ "${version}" =~ (2|3).* ]]; then suffix="-[0-9.]*" #List is separated by ":" hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-yarn-client${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-cli-*.jar:commons-io-*.jar" http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/docs/src/site/twiki/DG_QuickStart.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki index 2454985..37cb53c 100644 --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@ -41,10 +41,10 @@ suitable when same oozie package needs to be used in multiple set-ups with diffe 2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are available to customise the versions of the dependencies: --Dhadoop.version=<version> - default 2.4.0 +-Dhadoop.version=<version> - default 2.6.0 -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version --Dpig.version=<version> - default 0.12.1 +-Dpig.version=<version> - default 0.16.0 -Dpig.classifier=<classifier> - default h2 -Dsqoop.version=<version> - default 1.4.3 -Dsqoop.classifier=<classifier> - default hadoop100 http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki index f3fa863..f0bbe71 100644 --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@ -114,7 +114,7 @@ specified in the =test.properties= file (which is loaded by the =XTestCase= clas *hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-2) you wish to build Oozie against specifically. It will substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven. -For Hadoop 2.x, the default (and minimum) version is 2.4.0. +For Hadoop 2.x, the default (and minimum) version is 2.6.0. *generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated) @@ -216,7 +216,7 @@ the versions of the dependencies: -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version -Dpig.version=<version> - default 0.16.0 --Dpig.classifier=<classifier> - default none +-Dpig.classifier=<classifier> - default h2 -Dsqoop.version=<version> - default 1.4.3 -Dsqoop.classifier=<classifier> - default hadoop100 -Dtomcat.version=<version> - default 6.0.47 http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/findbugs-filter.xml ---------------------------------------------------------------------- diff --git a/findbugs-filter.xml b/findbugs-filter.xml index 53c8cb9..03ee4d1 100644 --- a/findbugs-filter.xml +++ b/findbugs-filter.xml @@ -16,14 +16,17 @@ limitations under the License. --> <FindBugsFilter> + <!-- excluded because eventServie is static and it cannot be simply rewritten --> <Match> <Class name="org.apache.oozie.command.XCommand" /> <Field name="eventService" /> <Bug pattern="MS_CANNOT_BE_FINAL" /> </Match> + <!-- excluded due to the complicated usage of keySet()/get() methods --> <Match> <Class name="org.apache.oozie.util.Instrumentation" /> + <Method name="toString" /> <Bug pattern="WMI_WRONG_MAP_ITERATOR" /> </Match> </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 81fdf79..24dbd4f 100644 --- a/pom.xml +++ b/pom.xml @@ -1946,32 +1946,6 @@ </properties> </profile> <profile> - <id>wfgen</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>workflowgenerator</module> - </modules> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <fork>true</fork> - <source>1.6</source> - <target>1.6</target> - <compilerArguments> - <Xlint/> - <Xmaxwarns>9999</Xmaxwarns> - </compilerArguments> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>loginServerExample</id> <activation> <activeByDefault>false</activeByDefault> http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index 6440d83..9e2ae10 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -261,8 +261,6 @@ public class Hive2Main extends LauncherMain { BeeLine beeLine = new BeeLine(); beeLine.setErrorStream(new PrintStream(new TeeOutputStream(System.err, new FileOutputStream(logFile)))); int status = beeLine.begin(args, null); - System.out.println("*** Return value from Beeline is: " + status); - System.out.println("*** Current security manager: " + System.getSecurityManager()); beeLine.close(); if (status != 0) { System.exit(status); http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java index 22d12ec..8e7b258 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java @@ -60,9 +60,8 @@ public class HdfsOperations { System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: " + new Path(actionDir, LauncherAM.ACTION_DATA_SEQUENCE_FILE).toUri()); - SequenceFile.Writer wr = null; - try { - wr = seqFileWriterFactory.createSequenceFileWriter(launcherJobConf, finalPath, Text.class, Text.class); + try (SequenceFile.Writer wr = + seqFileWriterFactory.createSequenceFileWriter(launcherJobConf, finalPath, Text.class, Text.class)) { if (wr != null) { for (Entry<String, String> entry : actionData.entrySet()) { @@ -71,14 +70,6 @@ public class HdfsOperations { } else { throw new IOException("SequenceFile.Writer is null for " + finalPath); } - } finally { - if (wr != null) { - try { - wr.close(); - } catch (IOException e) { - throw e; - } - } } return null; @@ -101,10 +92,11 @@ public class HdfsOperations { ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { - FileSystem fs = FileSystem.get(path.toUri(), conf); - java.io.Writer writer = new OutputStreamWriter(fs.create(path), DEFAULT_CHARSET); - writer.write(contents); - writer.close(); + try (FileSystem fs = FileSystem.get(path.toUri(), conf); + java.io.Writer writer = new OutputStreamWriter(fs.create(path), DEFAULT_CHARSET)) { + writer.write(contents); + } + return null; } }); @@ -114,17 +106,17 @@ public class HdfsOperations { return ugi.doAs(new PrivilegedExceptionAction<String>() { @Override public String run() throws Exception { - FileSystem fs = FileSystem.get(path.toUri(), conf); - InputStream is = fs.open(path); - BufferedReader reader = new BufferedReader(new InputStreamReader(is, DEFAULT_CHARSET)); StringBuilder sb = new StringBuilder(); - String contents; - while ((contents = reader.readLine()) != null) { - sb.append(contents); - } + try (FileSystem fs = FileSystem.get(path.toUri(), conf); + InputStream is = fs.open(path); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, DEFAULT_CHARSET))) { - reader.close(); + String contents; + while ((contents = reader.readLine()) != null) { + sb.append(contents); + } + } return sb.toString(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 20f6676..2b55997 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -48,10 +48,6 @@ import com.google.common.base.Preconditions; public class LauncherAM { private static final String OOZIE_ACTION_CONF_XML = "oozie.action.conf.xml"; private static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id"; - public static final String ACTIONOUTPUTTYPE_ID_SWAP = "IdSwap"; - public static final String ACTIONOUTPUTTYPE_OUTPUT = "Output"; - public static final String ACTIONOUTPUTTYPE_STATS = "Stats"; - public static final String ACTIONOUTPUTTYPE_EXT_CHILD_ID = "ExtChildID"; public static final String JAVA_CLASS_PATH = "java.class.path"; public static final String OOZIE_ACTION_ID = "oozie.action.id"; @@ -154,7 +150,7 @@ public class LauncherAM { public void run() throws Exception { final ErrorHolder errorHolder = new ErrorHolder(); OozieActionResult actionResult = OozieActionResult.FAILED; - boolean launcerExecutedProperly = false; + boolean launcherExecutedProperly = false; boolean backgroundAction = false; try { @@ -173,9 +169,9 @@ public class LauncherAM { final String[] mainArgs = getMainArguments(launcherJobConf); printDebugInfo(); setupMainConfiguration(); - launcerExecutedProperly = runActionMain(mainArgs, errorHolder, ugi); + launcherExecutedProperly = runActionMain(mainArgs, errorHolder, ugi); - if (launcerExecutedProperly) { + if (launcherExecutedProperly) { handleActionData(); if (actionData.get(ACTION_DATA_OUTPUT_PROPS) != null) { System.out.println(); @@ -201,7 +197,7 @@ public class LauncherAM { System.err.println("Launcher AM execution failed"); e.printStackTrace(System.out); e.printStackTrace(System.err); - launcerExecutedProperly = false; + launcherExecutedProperly = false; if (!errorHolder.isPopulated()) { errorHolder.setErrorCause(e); errorHolder.setErrorMessage(e.getMessage()); @@ -211,11 +207,11 @@ public class LauncherAM { try { ErrorHolder callbackErrorHolder = callbackHandler.getError(); - if (launcerExecutedProperly) { + if (launcherExecutedProperly) { actionResult = backgroundAction ? OozieActionResult.RUNNING : OozieActionResult.SUCCEEDED; } - if (!launcerExecutedProperly) { + if (!launcherExecutedProperly) { updateActionDataWithFailure(errorHolder, actionData); } else if (callbackErrorHolder != null) { // async error from the callback actionResult = OozieActionResult.FAILED; @@ -456,23 +452,23 @@ public class LauncherAM { private void handleActionData() throws IOException { // external child IDs processActionData(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS, null, - ACTION_DATA_EXTERNAL_CHILD_IDS, -1, ACTIONOUTPUTTYPE_EXT_CHILD_ID); + ACTION_DATA_EXTERNAL_CHILD_IDS, -1); // external stats processActionData(ACTION_PREFIX + ACTION_DATA_STATS, CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, - ACTION_DATA_STATS, Integer.MAX_VALUE, ACTIONOUTPUTTYPE_STATS); + ACTION_DATA_STATS, Integer.MAX_VALUE); // output data processActionData(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, - ACTION_DATA_OUTPUT_PROPS, 2048, ACTIONOUTPUTTYPE_OUTPUT); + ACTION_DATA_OUTPUT_PROPS, 2048); // id swap processActionData(ACTION_PREFIX + ACTION_DATA_NEW_ID, null, - ACTION_DATA_NEW_ID, -1, ACTIONOUTPUTTYPE_ID_SWAP); + ACTION_DATA_NEW_ID, -1); } private void processActionData(String propertyName, String maxSizePropertyName, String actionDataPropertyName, - int maxSizeDefault, String type) throws IOException { + int maxSizeDefault) throws IOException { String propValue = System.getProperty(propertyName); int maxSize = maxSizeDefault; @@ -484,7 +480,7 @@ public class LauncherAM { File actionDataFile = new File(propValue); if (localFsOperations.fileExists(actionDataFile)) { actionData.put(actionDataPropertyName, localFsOperations.getLocalFileContentAsString(actionDataFile, - type, maxSize)); + actionDataPropertyName, maxSize)); } } } @@ -525,7 +521,7 @@ public class LauncherAM { File externalChildIDs = new File(externalChildIdsProp); if (localFsOperations.fileExists(externalChildIDs)) { actionData.put(LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS, - localFsOperations.getLocalFileContentAsString(externalChildIDs, ACTIONOUTPUTTYPE_EXT_CHILD_ID, -1)); + localFsOperations.getLocalFileContentAsString(externalChildIDs, ACTION_DATA_EXTERNAL_CHILD_IDS, -1)); } } } catch (IOException ioe) { http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index 16cf4b1..59227d9 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -151,11 +151,7 @@ public class MapReduceMain extends LauncherMain { Class<? extends OozieActionConfigurator> actionConfiguratorKlass = klass.asSubclass(OozieActionConfigurator.class); OozieActionConfigurator actionConfigurator = actionConfiguratorKlass.newInstance(); actionConfigurator.configure(actionConf); - } catch (ClassNotFoundException e) { - throw new OozieActionConfiguratorException("An Exception occured while instantiating the action config class", e); - } catch (InstantiationException e) { - throw new OozieActionConfiguratorException("An Exception occured while instantiating the action config class", e); - } catch (IllegalAccessException e) { + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { throw new OozieActionConfiguratorException("An Exception occured while instantiating the action config class", e); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java index f6af98b..9cdedb7 100644 --- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java @@ -17,10 +17,6 @@ */ package org.apache.oozie.action.hadoop; -import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_EXT_CHILD_ID; -import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_ID_SWAP; -import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_OUTPUT; -import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_STATS; import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS; import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_NEW_ID; import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_OUTPUT_PROPS; @@ -388,7 +384,7 @@ public class TestLauncherAM { @Test public void testUpdateActionDataFailsWithActionError() throws Exception { setupActionOutputContents(); - given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_EXT_CHILD_ID), anyInt())) + given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_EXTERNAL_CHILD_IDS), anyInt())) .willThrow(new IOException()); thrown.expect(IOException.class); @@ -498,17 +494,17 @@ public class TestLauncherAM { private void setupActionOutputContents() throws IOException { // output files generated by an action - given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_EXT_CHILD_ID), anyInt())) - .willReturn(ACTIONOUTPUTTYPE_EXT_CHILD_ID); + given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_EXTERNAL_CHILD_IDS), anyInt())) + .willReturn(ACTION_DATA_EXTERNAL_CHILD_IDS); - given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_ID_SWAP), anyInt())) - .willReturn(ACTIONOUTPUTTYPE_ID_SWAP); + given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_NEW_ID), anyInt())) + .willReturn(ACTION_DATA_NEW_ID); - given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_OUTPUT), anyInt())) - .willReturn(ACTIONOUTPUTTYPE_OUTPUT); + given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_OUTPUT_PROPS), anyInt())) + .willReturn(ACTION_DATA_OUTPUT_PROPS); - given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_STATS), anyInt())) - .willReturn(ACTIONOUTPUTTYPE_STATS); + given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_STATS), anyInt())) + .willReturn(ACTION_DATA_STATS); } private void setupArgsForMainClass(final String... args) { @@ -546,10 +542,10 @@ public class TestLauncherAM { String output = actionData.get(ACTION_DATA_OUTPUT_PROPS); String idSwap = actionData.get(ACTION_DATA_NEW_ID); - assertThat("extChildID output", ACTIONOUTPUTTYPE_EXT_CHILD_ID, equalTo(extChildId)); - assertThat("stats output", ACTIONOUTPUTTYPE_STATS, equalTo(stats)); - assertThat("action output", ACTIONOUTPUTTYPE_OUTPUT, equalTo(output)); - assertThat("idSwap output", ACTIONOUTPUTTYPE_ID_SWAP, equalTo(idSwap)); + assertThat("extChildID output", ACTION_DATA_EXTERNAL_CHILD_IDS, equalTo(extChildId)); + assertThat("stats output", ACTION_DATA_STATS, equalTo(stats)); + assertThat("action output", ACTION_DATA_OUTPUT_PROPS, equalTo(output)); + assertThat("idSwap output", ACTION_DATA_NEW_ID, equalTo(idSwap)); } private void assertNoActionOutputData() { http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/spark/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/spark/pom.xml b/sharelib/spark/pom.xml index df17879..dec505b 100644 --- a/sharelib/spark/pom.xml +++ b/sharelib/spark/pom.xml @@ -251,7 +251,7 @@ <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-yarn_2.10</artifactId> + <artifactId>spark-yarn_${spark.scala.binary.version}</artifactId> <version>${spark.version}</version> <scope>compile</scope> <exclusions> @@ -374,25 +374,6 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <configuration> - <target> - <!-- needed to include Main class in classpath for mini yarn cluster for unit tests --> - <echo file="${project.build.directory}/test-classes/mrapp-generated-classpath" - append="true" message=":${project.build.directory}/classes"/> - </target> - </configuration> - <goals> - <goal>run</goal> - </goals> - <phase>generate-test-resources</phase> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> <configuration> <finalName>partial-sharelib</finalName>
