Repository: oozie Updated Branches: refs/heads/master 533e6f60a -> 5998c18fd
OOZIE-2944 Shell action example does not work with Oozie on Yarn on hadoop 2.6 (asasvari via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5998c18f Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5998c18f Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5998c18f Branch: refs/heads/master Commit: 5998c18fde1da769e91e3ef1bcca484723730c76 Parents: 533e6f6 Author: Gezapeti Cseh <[email protected]> Authored: Thu Jun 15 20:43:21 2017 +0200 Committer: Gezapeti Cseh <[email protected]> Committed: Thu Jun 15 20:43:21 2017 +0200 ---------------------------------------------------------------------- .../oozie/action/email/EmailActionExecutor.java | 2 +- .../oozie/action/hadoop/FsActionExecutor.java | 6 +- .../oozie/action/hadoop/FsELFunctions.java | 4 +- .../oozie/action/hadoop/HadoopELFunctions.java | 2 +- .../oozie/action/hadoop/JavaActionExecutor.java | 5 +- .../command/bundle/BundleSubmitXCommand.java | 4 +- .../command/coord/CoordSubmitXCommand.java | 4 +- .../apache/oozie/command/wf/ActionXCommand.java | 2 +- .../apache/oozie/command/wf/ReRunXCommand.java | 2 +- .../apache/oozie/command/wf/SubmitXCommand.java | 2 +- .../apache/oozie/command/wf/WfEndXCommand.java | 2 +- .../apache/oozie/dependency/FSURIHandler.java | 2 +- .../oozie/service/AuthorizationService.java | 4 +- .../oozie/service/HCatAccessorService.java | 2 +- .../oozie/service/HadoopAccessorService.java | 16 ++--- .../apache/oozie/service/ShareLibService.java | 8 +-- .../oozie/service/WorkflowAppService.java | 7 +- .../apache/oozie/servlet/V2ValidateServlet.java | 2 +- .../java/org/apache/oozie/util/JobUtils.java | 4 +- .../action/hadoop/TestJavaActionExecutor.java | 8 +-- .../oozie/action/hadoop/TestLauncher.java | 73 ++++++++++---------- .../wf/TestWorkflowActionKillXCommand.java | 7 +- .../oozie/service/TestAuthorizationService.java | 4 +- .../oozie/service/TestHAShareLibService.java | 2 +- .../service/TestHadoopAccessorService.java | 14 ++-- .../java/org/apache/oozie/test/XFsTestCase.java | 9 +-- .../java/org/apache/oozie/test/XTestCase.java | 2 +- release-log.txt | 1 + .../oozie/action/hadoop/LauncherMapper.java | 60 ++++++++-------- .../apache/oozie/tools/OozieSharelibCLI.java | 4 +- .../oozie/tools/TestOozieSharelibCLI.java | 2 +- 31 files changed, 125 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/action/email/EmailActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/email/EmailActionExecutor.java b/core/src/main/java/org/apache/oozie/action/email/EmailActionExecutor.java index d55b3af..d941753 100644 --- a/core/src/main/java/org/apache/oozie/action/email/EmailActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/email/EmailActionExecutor.java @@ -346,7 +346,7 @@ public class EmailActionExecutor extends ActionExecutor { URI uri; public URIDataSource(URI uri, String user) throws HadoopAccessorException { this.uri = uri; - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); fs = has.createFileSystem(user, uri, fsConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java index 7f7c676..e1150f7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -34,7 +35,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.oozie.action.ActionExecutor; @@ -316,7 +316,7 @@ public class FsActionExecutor extends ActionExecutor { private FileSystem getFileSystemFor(Path path, Context context, XConfiguration fsConf) throws HadoopAccessorException { String user = context.getWorkflow().getUser(); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf conf = has.createJobConf(path.toUri().getAuthority()); + Configuration conf = has.createConfiguration(path.toUri().getAuthority()); XConfiguration.copy(context.getProtoActionConf(), conf); if (fsConf != null) { XConfiguration.copy(fsConf, conf); @@ -332,7 +332,7 @@ public class FsActionExecutor extends ActionExecutor { */ private FileSystem getFileSystemFor(Path path, String user) throws HadoopAccessorException { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf jobConf = has.createJobConf(path.toUri().getAuthority()); + Configuration jobConf = has.createConfiguration(path.toUri().getAuthority()); return has.createFileSystem(user, path.toUri(), jobConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java index c16f560..295f81a 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java @@ -22,11 +22,11 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.DagELFunctions; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowJob; @@ -44,7 +44,7 @@ public class FsELFunctions { WorkflowJob workflow = DagELFunctions.getWorkflow(); String user = workflow.getUser(); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf conf = has.createJobConf(uri.getAuthority()); + Configuration conf = has.createConfiguration(uri.getAuthority()); return has.createFileSystem(user, uri, conf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java index 777c187..0b8f30f 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java @@ -59,7 +59,7 @@ public class HadoopELFunctions { public static String hadoop_conf(String hadoopConfHostPort, String propName) { Configuration conf = Services.get().get(HadoopAccessorService.class) - .createJobConf(hadoopConfHostPort); + .createConfiguration(hadoopConfHostPort); String prop = conf.get(propName); if (prop == null || prop.equals("")) { conf = new Configuration(); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 56eb035..2c8d52a 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -57,7 +57,6 @@ import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.DiskChecker; @@ -246,7 +245,7 @@ public class JavaActionExecutor extends ActionExecutor { String nameNode = actionXml.getChild("name-node", ns).getTextTrim(); Configuration conf = null; if (loadResources) { - conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker); + conf = Services.get().get(HadoopAccessorService.class).createConfiguration(jobTracker); } else { conf = new Configuration(false); @@ -339,7 +338,7 @@ public class JavaActionExecutor extends ActionExecutor { else { if (path.toUri().getAuthority() != null) { fs = has.createFileSystem(context.getWorkflow().getUser(), path.toUri(), - has.createJobConf(path.toUri().getAuthority())); + has.createConfiguration(path.toUri().getAuthority())); } else { fs = context.getAppFileSystem(); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java index 3df37fa..3a17234 100644 --- a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java @@ -250,7 +250,7 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { Path bundleAppPath = new Path(bundleAppPathStr); String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(bundleAppPath.toUri().getAuthority()); + Configuration fsConf = has.createConfiguration(bundleAppPath.toUri().getAuthority()); FileSystem fs = has.createFileSystem(user, bundleAppPath.toUri(), fsConf); // app path could be a directory @@ -307,7 +307,7 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { URI uri = new URI(appPath); LOG.debug("user =" + user); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path appDefPath = null; http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java index 969336d..17e7fa0 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java @@ -555,7 +555,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { Path coordAppPath = new Path(coordAppPathStr); String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(coordAppPath.toUri().getAuthority()); + Configuration fsConf = has.createConfiguration(coordAppPath.toUri().getAuthority()); FileSystem fs = has.createFileSystem(user, coordAppPath.toUri(), fsConf); // app path could be a directory @@ -1174,7 +1174,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { URI uri = new URI(appPath); LOG.debug("user =" + user); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path appDefPath = null; http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java index 4f127c0..432b077 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java @@ -520,7 +520,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { WorkflowJob workflow = getWorkflow(); URI uri = new URI(getWorkflow().getAppPath()); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); return has.createFileSystem(workflow.getUser(), uri, fsConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/command/wf/ReRunXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/ReRunXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ReRunXCommand.java index 178eaf4..178ae27 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/ReRunXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/ReRunXCommand.java @@ -151,7 +151,7 @@ public class ReRunXCommand extends WorkflowXCommand<Void> { appPath = conf.get(OozieClient.APP_PATH); URI uri = new URI(appPath); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(wfBean.getUser(), uri, fsConf); Path configDefault = null; http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java index 3e2996f..54b1d46 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java @@ -137,7 +137,7 @@ public class SubmitXCommand extends WorkflowXCommand<String> { String user = conf.get(OozieClient.USER_NAME); URI uri = new URI(conf.get(OozieClient.APP_PATH)); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path configDefault = null; http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/command/wf/WfEndXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/WfEndXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/WfEndXCommand.java index 7fb8e90..e282d94 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/WfEndXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/WfEndXCommand.java @@ -80,7 +80,7 @@ public class WfEndXCommand extends WorkflowXCommand<Void> { URISyntaxException { URI uri = new URI(workflow.getAppPath()); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); return has.createFileSystem(workflow.getUser(), uri, fsConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java b/core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java index 2cfcbab..2b8788c 100644 --- a/core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java +++ b/core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java @@ -180,7 +180,7 @@ public class FSURIHandler implements URIHandler { if (user == null) { throw new HadoopAccessorException(ErrorCode.E0902, "user has to be specified to access FileSystem"); } - Configuration fsConf = service.createJobConf(uri.getAuthority()); + Configuration fsConf = service.createConfiguration(uri.getAuthority()); return service.createFileSystem(user, uri, fsConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/service/AuthorizationService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/AuthorizationService.java b/core/src/main/java/org/apache/oozie/service/AuthorizationService.java index 4e4ab17..f4ccf4f 100644 --- a/core/src/main/java/org/apache/oozie/service/AuthorizationService.java +++ b/core/src/main/java/org/apache/oozie/service/AuthorizationService.java @@ -318,7 +318,7 @@ public class AuthorizationService implements Service { try { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = new Path(appPath).toUri(); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path path = new Path(appPath); @@ -370,7 +370,7 @@ public class AuthorizationService implements Service { try { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = new Path(appPath).toUri(); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path path = new Path(appPath); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java b/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java index 808eb6f..ba98cf5 100644 --- a/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java @@ -94,7 +94,7 @@ public class HCatAccessorService implements Service { HadoopAccessorService has = services.get(HadoopAccessorService.class); try { FileSystem fs = has.createFileSystem( - System.getProperty("user.name"), p.toUri(), has.createJobConf(p.toUri().getAuthority())); + System.getProperty("user.name"), p.toUri(), has.createConfiguration(p.toUri().getAuthority())); if (fs.exists(p)) { FSDataInputStream is = null; try { http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 9624104..7f1145d 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -338,20 +338,20 @@ public class HadoopAccessorService implements Service { } /** - * Creates a JobConf using the site configuration for the specified hostname:port. + * Creates a Configuration using the site configuration for the specified hostname:port. * <p> * If the specified hostname:port is not defined it falls back to the '*' site * configuration if available. If the '*' site configuration is not available, * the JobConf has all Hadoop defaults. * * @param hostPort hostname:port to lookup Hadoop site configuration. - * @return a JobConf with the corresponding site configuration for hostPort. + * @return a Configuration with the corresponding site configuration for hostPort. */ - public JobConf createJobConf(String hostPort) { - JobConf jobConf = new JobConf(getCachedConf()); - XConfiguration.copy(getConfiguration(hostPort), jobConf); - jobConf.setBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, true); - return jobConf; + public Configuration createConfiguration(String hostPort) { + Configuration appConf = new Configuration(getCachedConf()); + XConfiguration.copy(getConfiguration(hostPort), appConf); + appConf.setBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, true); + return appConf; } public Configuration getCachedConf() { @@ -774,4 +774,4 @@ public class HadoopAccessorService implements Service { return localResource; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index a4d2c02..82a0aff 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -121,7 +121,7 @@ public class ShareLibService implements Service, Instrumentable { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = launcherlibPath.toUri(); try { - fs = FileSystem.get(has.createJobConf(uri.getAuthority())); + fs = FileSystem.get(has.createConfiguration(uri.getAuthority())); //cache action key sharelib conf list cacheActionKeySharelibConfList(); updateLauncherLib(); @@ -174,7 +174,7 @@ public class ShareLibService implements Service, Instrumentable { Path launcherlibPath = getLauncherlibPath(); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = launcherlibPath.toUri(); - fs = FileSystem.get(has.createJobConf(uri.getAuthority())); + fs = FileSystem.get(has.createConfiguration(uri.getAuthority())); } Path launcherlibPath = getLauncherlibPath(); setupLauncherLibPath(fs, launcherlibPath); @@ -527,7 +527,7 @@ public class ShareLibService implements Service, Instrumentable { Path launcherlibPath = getLauncherlibPath(); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = launcherlibPath.toUri(); - fs = FileSystem.get(has.createJobConf(uri.getAuthority())); + fs = FileSystem.get(has.createConfiguration(uri.getAuthority())); } Map<String, List<Path>> tempShareLibMap = new HashMap<String, List<Path>>(); @@ -613,7 +613,7 @@ public class ShareLibService implements Service, Instrumentable { Path shareFileMappingPath = new Path(sharelibFileMapping); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - FileSystem filesystem = FileSystem.get(has.createJobConf(shareFileMappingPath.toUri().getAuthority())); + FileSystem filesystem = FileSystem.get(has.createConfiguration(shareFileMappingPath.toUri().getAuthority())); Properties prop = new Properties(); prop.load(filesystem.open(new Path(sharelibFileMapping))); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java index 6891bbe..841bcd7 100644 --- a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java +++ b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java @@ -23,7 +23,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.client.OozieClient; import org.apache.oozie.workflow.WorkflowApp; import org.apache.oozie.workflow.WorkflowException; @@ -116,8 +115,8 @@ public abstract class WorkflowAppService implements Service { try { URI uri = new URI(appPath); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf jobConf = has.createJobConf(uri.getAuthority()); - FileSystem fs = has.createFileSystem(user, uri, jobConf); + Configuration appConf = has.createConfiguration(uri.getAuthority()); + FileSystem fs = has.createFileSystem(user, uri, appConf); // app path could be a directory Path path = new Path(uri.getPath()); @@ -167,7 +166,7 @@ public abstract class WorkflowAppService implements Service { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = new URI(jobConf.get(OozieClient.APP_PATH)); - Configuration conf = has.createJobConf(uri.getAuthority()); + Configuration conf = has.createConfiguration(uri.getAuthority()); XConfiguration protoConf = new XConfiguration(); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java index dbb3d49..45bb1f0 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java @@ -77,7 +77,7 @@ public class V2ValidateServlet extends JsonRestServlet { try { URI uri = new URI(file); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path path = new Path(uri.getPath()); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/main/java/org/apache/oozie/util/JobUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java index 5d948d9..681b58a 100644 --- a/core/src/main/java/org/apache/oozie/util/JobUtils.java +++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java @@ -68,7 +68,7 @@ public class JobUtils { try { URI uri = new Path(appPathStr).toUri(); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); fs = has.createFileSystem(user, uri, fsConf); } catch (HadoopAccessorException ex) { @@ -147,7 +147,7 @@ public class JobUtils { public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException { if (fs == null) { Configuration defaultConf = Services.get().get(HadoopAccessorService.class) - .createJobConf(conf.get(JavaActionExecutor.HADOOP_YARN_RM)); + .createConfiguration(conf.get(JavaActionExecutor.HADOOP_YARN_RM)); XConfiguration.copy(conf, defaultConf); // it fails with conf, therefore we pass defaultConf instead fs = file.getFileSystem(defaultConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 97cd469..67c7d06 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -31,14 +31,11 @@ import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Properties; -import java.util.Set; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.examples.SleepJob; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -46,10 +43,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.action.ActionExecutor; @@ -63,7 +58,6 @@ import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.UUIDService; -import org.apache.oozie.service.UserGroupInformationService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.WorkflowStoreService; import org.apache.oozie.util.IOUtils; @@ -1111,7 +1105,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "start")). addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class)); Configuration conf = Services.get().get(HadoopAccessorService.class). - createJobConf(new URI(getNameNodeUri()).getAuthority()); + createConfiguration(new URI(getNameNodeUri()).getAuthority()); conf.set(OozieClient.APP_PATH, getNameNodeUri() + "/testPath"); conf.set(OozieClient.LOG_TOKEN, "testToken"); conf.set(OozieClient.USER_NAME, getTestUser()); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java index 1088fd1..1b7724e 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java @@ -70,34 +70,33 @@ public class TestLauncher extends XFsTestCase { Path launcherJar = new Path(actionDir, "launcher.jar"); fs.copyFromLocalFile(new Path(jar.toString()), launcherJar); - JobConf jobConf = Services.get().get(HadoopAccessorService.class). - createJobConf(new URI(getNameNodeUri()).getAuthority()); -// jobConf.setJar(jar.getAbsolutePath()); - jobConf.set("user.name", getTestUser()); - jobConf.setInt("mapred.map.tasks", 1); - jobConf.setInt("mapred.map.max.attempts", 1); - jobConf.setInt("mapred.reduce.max.attempts", 1); + Configuration appConf = Services.get().get(HadoopAccessorService.class). + createConfiguration(new URI(getNameNodeUri()).getAuthority()); + appConf.set("user.name", getTestUser()); + appConf.setInt("mapred.map.tasks", 1); + appConf.setInt("mapred.map.max.attempts", 1); + appConf.setInt("mapred.reduce.max.attempts", 1); - jobConf.set("mapreduce.framework.name", "yarn"); - jobConf.set("mapred.job.tracker", getJobTrackerUri()); - jobConf.set("fs.default.name", getNameNodeUri()); + appConf.set("mapreduce.framework.name", "yarn"); + appConf.set("mapred.job.tracker", getJobTrackerUri()); + appConf.set("fs.default.name", getNameNodeUri()); - LauncherHelper.setupMainClass(jobConf, LauncherMainTester.class.getName()); - LauncherHelper.setupMainArguments(jobConf, arg); + LauncherHelper.setupMainClass(appConf, LauncherMainTester.class.getName()); + LauncherHelper.setupMainArguments(appConf, arg); Configuration actionConf = new XConfiguration(); - LauncherHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); - LauncherHelper.setupYarnRestartHandling(jobConf, jobConf, "1@a", System.currentTimeMillis()); + LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); + LauncherHelper.setupYarnRestartHandling(appConf, appConf, "1@a", System.currentTimeMillis()); assertEquals("1", actionConf.get("oozie.job.id")); assertEquals("1@a", actionConf.get("oozie.action.id")); - DistributedCache.addFileToClassPath(new Path(launcherJar.toUri().getPath()), jobConf); + DistributedCache.addFileToClassPath(new Path(launcherJar.toUri().getPath()), appConf); JobClient jobClient = createJobClient(); - final RunningJob runningJob = jobClient.submitJob(jobConf); + final RunningJob runningJob = jobClient.submitJob(new JobConf(appConf)); System.out.println("Action Dir: " + actionDir); System.out.println("LauncherMapper ID: " + runningJob.getJobID().toString()); @@ -108,7 +107,7 @@ public class TestLauncher extends XFsTestCase { } }); - assertTrue(jobConf.get("oozie.action.prepare.xml").equals("")); + assertTrue(appConf.get("oozie.action.prepare.xml").equals("")); return runningJob; } @@ -304,15 +303,15 @@ public class TestLauncher extends XFsTestCase { Path actionDir = getFsTestCaseDir(); // Setting up the job configuration - JobConf jobConf = Services.get().get(HadoopAccessorService.class). - createJobConf(new URI(getNameNodeUri()).getAuthority()); - jobConf.set("user.name", getTestUser()); - jobConf.set("fs.default.name", getNameNodeUri()); + Configuration appConf = Services.get().get(HadoopAccessorService.class). + createConfiguration(new URI(getNameNodeUri()).getAuthority()); + appConf.set("user.name", getTestUser()); + appConf.set("fs.default.name", getNameNodeUri()); Configuration actionConf = new XConfiguration(); String prepareBlock = ""; - LauncherHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, prepareBlock); - assertTrue(jobConf.get("oozie.action.prepare.xml").equals("")); + LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, prepareBlock); + assertTrue(appConf.get("oozie.action.prepare.xml").equals("")); } // Test to ensure that the property value "oozie.action.prepare.xml" in the configuration of the job is properly set @@ -323,15 +322,15 @@ public class TestLauncher extends XFsTestCase { Path newDir = new Path(actionDir, "newDir"); // Setting up the job configuration - JobConf jobConf = Services.get().get(HadoopAccessorService.class). - createJobConf(new URI(getNameNodeUri()).getAuthority()); - jobConf.set("user.name", getTestUser()); - jobConf.set("fs.default.name", getNameNodeUri()); + Configuration appConf = Services.get().get(HadoopAccessorService.class). + createConfiguration(new URI(getNameNodeUri()).getAuthority()); + appConf.set("user.name", getTestUser()); + appConf.set("fs.default.name", getNameNodeUri()); Configuration actionConf = new XConfiguration(); String prepareBlock = "<prepare>" + "<mkdir path='" + newDir + "'/>" + "</prepare>"; - LauncherHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, prepareBlock); - assertTrue(jobConf.get("oozie.action.prepare.xml").equals(prepareBlock)); + LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, prepareBlock); + assertTrue(appConf.get("oozie.action.prepare.xml").equals(prepareBlock)); } public void testSetupMainClass() throws Exception { @@ -360,22 +359,22 @@ public class TestLauncher extends XFsTestCase { public void testSetupLauncherInfoHadoop2_0_2_alphaWorkaround() throws Exception { Path actionDir = getFsTestCaseDir(); // Setting up the job configuration - JobConf jobConf = Services.get().get(HadoopAccessorService.class). - createJobConf(new URI(getNameNodeUri()).getAuthority()); - jobConf.set("user.name", getTestUser()); - jobConf.set("fs.default.name", getNameNodeUri()); + Configuration appConf = Services.get().get(HadoopAccessorService.class). + createConfiguration(new URI(getNameNodeUri()).getAuthority()); + appConf.set("user.name", getTestUser()); + appConf.set("fs.default.name", getNameNodeUri()); Configuration actionConf = new XConfiguration(); actionConf.set("mapreduce.job.cache.files", "a.jar,aa.jar#aa.jar"); - LauncherHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); - assertFalse(jobConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)); + LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); + assertFalse(appConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)); assertEquals("a.jar,aa.jar#aa.jar", actionConf.get("mapreduce.job.cache.files")); Services.get().getConf().setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true); actionConf = new XConfiguration(); actionConf.set("mapreduce.job.cache.files", "a.jar,aa.jar#aa.jar"); - LauncherHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); - assertTrue(jobConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)); + LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); + assertTrue(appConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)); assertEquals("aa.jar#aa.jar", actionConf.get("mapreduce.job.cache.files")); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionKillXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionKillXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionKillXCommand.java index ef75f14..d68f656 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionKillXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionKillXCommand.java @@ -22,6 +22,7 @@ import java.net.URI; import java.util.Date; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.examples.SleepJob; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; @@ -160,8 +161,8 @@ public class TestWorkflowActionKillXCommand extends XDataTestCase { } private String launchSleepJob(int sleep) throws Exception { - JobConf jobConf = Services.get().get(HadoopAccessorService.class) - .createJobConf(new URI(getNameNodeUri()).getAuthority()); + Configuration jobConf = Services.get().get(HadoopAccessorService.class) + .createConfiguration(new URI(getNameNodeUri()).getAuthority()); JobClient jobClient = createJobClient(); SleepJob sleepjob = new SleepJob(); @@ -171,7 +172,7 @@ public class TestWorkflowActionKillXCommand extends XDataTestCase { jobConf.set(LauncherMain.MAPREDUCE_JOB_TAGS, "sleepjob"); System.setProperty(LauncherMain.OOZIE_JOB_LAUNCH_TIME, String.valueOf(System.currentTimeMillis())); - jobClient.submitJob(jobConf); + jobClient.submitJob(new JobConf(jobConf)); Set<ApplicationId> apps = Sets.newHashSet(); apps = LauncherMain.getChildYarnJobs(jobConf, ApplicationsRequestScope.ALL); assertEquals("Number of YARN apps", apps.size(), 1); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java b/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java index b5e527e..c32bfc2 100644 --- a/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java @@ -144,7 +144,7 @@ public class TestAuthorizationService extends XDataTestCase { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = getFileSystem().getUri(); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fileSystem = has.createFileSystem(getTestUser(), uri, fsConf); Path path = new Path(fileSystem.getWorkingDirectory(), UUID.randomUUID().toString()); @@ -227,7 +227,7 @@ public class TestAuthorizationService extends XDataTestCase { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = getFileSystem().getUri(); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fileSystem = has.createFileSystem(getTestUser(), uri, fsConf); try { http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java index 791f568..5087505 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java @@ -65,7 +65,7 @@ public class TestHAShareLibService extends ZKXTestCase { Path launcherlibPath = Services.get().get(WorkflowAppService.class).getSystemLibPath(); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = launcherlibPath.toUri(); - fs = FileSystem.get(has.createJobConf(uri.getAuthority())); + fs = FileSystem.get(has.createConfiguration(uri.getAuthority())); Date time = new Date(System.currentTimeMillis()); Path basePath = new Path(Services.get().getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java index a1ee004..960c2f9 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java @@ -105,9 +105,9 @@ public class TestHadoopAccessorService extends XFsTestCase { Services services = Services.get(); HadoopAccessorService has = services.get(HadoopAccessorService.class); assertNotNull(has); - assertNotNull(has.createJobConf("*")); - assertNotNull(has.createJobConf("jt")); - assertEquals("bar", has.createJobConf("jt").get("foo")); + assertNotNull(has.createConfiguration("*")); + assertNotNull(has.createConfiguration("jt")); + assertEquals("bar", has.createConfiguration("jt").get("foo")); assertNotNull(has.createActionDefaultConf("*", "action")); assertNotNull(has.createActionDefaultConf("jt", "action")); assertNotNull(has.createActionDefaultConf("jt", "actionx")); @@ -154,7 +154,7 @@ public class TestHadoopAccessorService extends XFsTestCase { public void testCreateJobClient() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf conf = has.createJobConf(getJobTrackerUri()); + Configuration conf = has.createConfiguration(getJobTrackerUri()); JobClient jc = has.createJobClient(getTestUser(), conf); assertNotNull(jc); @@ -173,7 +173,7 @@ public class TestHadoopAccessorService extends XFsTestCase { public void testCreateYarnClient() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf conf = has.createJobConf(getJobTrackerUri()); + Configuration conf = has.createConfiguration(getJobTrackerUri()); YarnClient yc = has.createYarnClient(getTestUser(), conf); assertNotNull(yc); @@ -201,7 +201,7 @@ public class TestHadoopAccessorService extends XFsTestCase { public void testCreateFileSystem() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); - JobConf conf = has.createJobConf(getJobTrackerUri()); + Configuration conf = has.createConfiguration(getJobTrackerUri()); FileSystem fs = has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), conf); assertNotNull(fs); @@ -344,7 +344,7 @@ public class TestHadoopAccessorService extends XFsTestCase { public void testCreateLocalResourceForConfigurationFile() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); String filename = "foo.xml"; - Configuration conf = has.createJobConf(getNameNodeUri()); + Configuration conf = has.createConfiguration(getNameNodeUri()); conf.set("foo", "bar"); LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(), getFsTestCaseDir()); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/test/XFsTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XFsTestCase.java b/core/src/test/java/org/apache/oozie/test/XFsTestCase.java index b017b41..2b33d93 100644 --- a/core/src/test/java/org/apache/oozie/test/XFsTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XFsTestCase.java @@ -25,10 +25,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XLog; -import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext; import org.apache.oozie.service.HadoopAccessorException; @@ -37,7 +35,6 @@ import org.apache.oozie.service.Services; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; /** * Base JUnit <code>TestCase</code> subclass used by all Oozie testcases that need Hadoop FS access. <p/> As part of its @@ -77,7 +74,7 @@ public abstract class XFsTestCase extends XTestCase { has = new HadoopAccessorService(); has.init(conf); - JobConf jobConf = has.createJobConf(getNameNodeUri()); + Configuration jobConf = has.createConfiguration(getNameNodeUri()); XConfiguration.copy(conf, jobConf); fileSystem = has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), jobConf); fsTestDir = initFileSystem(fileSystem); @@ -173,7 +170,7 @@ public abstract class XFsTestCase extends XTestCase { * @throws HadoopAccessorException thrown if the JobClient could not be obtained. */ protected JobClient createJobClient() throws HadoopAccessorException { - JobConf conf = has.createJobConf(getJobTrackerUri()); + Configuration conf = has.createConfiguration(getJobTrackerUri()); conf.set("mapred.job.tracker", getJobTrackerUri()); conf.set("fs.default.name", getNameNodeUri()); @@ -185,7 +182,7 @@ public abstract class XFsTestCase extends XTestCase { * Example: /user/test/oozie-abcd/0000003-160913132555310-oozie-abcd-W/hadoop--map-reduce/launcher.xml * * @param userName current username - * @param action workflow Action object + * @param job workflow Action object * @param services Oozie Services class * @param context Executor context * @param fileName the filename http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index d7ce9b2..caa6132 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -1235,7 +1235,7 @@ public abstract class XTestCase extends TestCase { final ApplicationId appId = ConverterUtils.toApplicationId(externalId); final MutableObject<YarnApplicationState> finalState = new MutableObject<YarnApplicationState>(); - Configuration conf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri()); + Configuration conf = Services.get().get(HadoopAccessorService.class).createConfiguration(getJobTrackerUri()); final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), conf); try { http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 237f08d..e334a90 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2944 Shell action example does not work with Oozie on Yarn on hadoop 2.6 (asasvari via gezapeti) OOZIE-2933 Switch from Findbugs to Spotbugs (Jan Hentschel via gezapeti) OOZIE-2651 Set javax.xml.parsers.DocumentBuilderFactory sys prop to make XML handling faster (rkanter) OOZIE-2938 Fix Findbugs warnings in oozie-sharelib-hive module (Jan Hentschel via gezapeti) http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java index 912eba2..d0d3629 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java @@ -45,8 +45,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.xml.sax.SAXException; @@ -55,7 +53,7 @@ import com.google.common.base.Strings; import javax.xml.parsers.ParserConfigurationException; // TODO: OYA: Delete :) -public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, Runnable { +public class LauncherMapper<K1, V1, K2, V2> implements Runnable { static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class"; @@ -122,7 +120,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R } } - private JobConf jobConf; + private Configuration appConf; private Path actionDir; private ScheduledThreadPoolExecutor timer; @@ -134,16 +132,15 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R actionData = new HashMap<String,String>(); } - @Override - public void configure(JobConf jobConf) { + public void configure(Configuration appConf) { System.out.println(); System.out.println("Oozie Launcher starts"); System.out.println(); - this.jobConf = jobConf; - actionDir = new Path(getJobConf().get(OOZIE_ACTION_DIR_PATH)); - String recoveryId = jobConf.get(OOZIE_ACTION_RECOVERY_ID, null); + this.appConf = appConf; + actionDir = new Path(getAppConf().get(OOZIE_ACTION_DIR_PATH)); + String recoveryId = appConf.get(OOZIE_ACTION_RECOVERY_ID, null); try { - setRecoveryId(jobConf, actionDir, recoveryId); + setRecoveryId(appConf, actionDir, recoveryId); } catch (LauncherException ex) { System.out.println("Launcher config error "+ex.getMessage()); @@ -152,7 +149,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R } } - @Override + public void map(K1 key, V1 value, OutputCollector<K2, V2> collector, Reporter reporter) throws IOException { SecurityManager initialSecurityManager = System.getSecurityManager(); try { @@ -160,8 +157,8 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R throw configureFailureEx; } else { - String mainClass = getJobConf().get(CONF_OOZIE_ACTION_MAIN_CLASS); - if (getJobConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) { + String mainClass = getAppConf().get(CONF_OOZIE_ACTION_MAIN_CLASS); + if (getAppConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) { System.err.println("WARNING, workaround for Hadoop 2.0.2-alpha distributed cached issue (MAPREDUCE-4820) enabled"); } String msgPrefix = "Main class [" + mainClass + "], "; @@ -193,7 +190,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R throw new LauncherException(ex.getMessage(), ex); } - String[] args = getMainArguments(getJobConf()); + String[] args = getMainArguments(getAppConf()); printContentsOfCurrentDir(); @@ -214,7 +211,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R System.out.println("Main class : " + mainClass); System.out.println(); System.out.println("Maximum output : " - + getJobConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024)); + + getAppConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024)); System.out.println(); printArgs("Arguments :", args); @@ -233,7 +230,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R System.out.flush(); try { - Class klass = getJobConf().getClass(CONF_OOZIE_ACTION_MAIN_CLASS, Object.class); + Class klass = getAppConf().getClass(CONF_OOZIE_ACTION_MAIN_CLASS, Object.class); Method mainMethod = klass.getMethod("main", String[].class); mainMethod.invoke(null, (Object) args); } @@ -326,7 +323,6 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R } } - @Override public void close() throws IOException { System.out.println(); System.out.println("Oozie Launcher ends"); @@ -361,8 +357,8 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R Configuration.addDefaultResource(PROPAGATION_CONF_XML); } - protected JobConf getJobConf() { - return jobConf; + protected Configuration getAppConf() { + return appConf; } private void handleActionData() throws IOException, LauncherException { @@ -380,7 +376,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R if (statsProp != null) { File actionStatsData = new File(statsProp); if (actionStatsData.exists()) { - int statsMaxOutputData = getJobConf().getInt(CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, Integer.MAX_VALUE); + int statsMaxOutputData = getAppConf().getInt(CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, Integer.MAX_VALUE); actionData.put(ACTION_DATA_STATS, getLocalFileContentStr(actionStatsData, "Stats", statsMaxOutputData)); } } @@ -390,7 +386,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R if (outputProp != null) { File actionOutputData = new File(outputProp); if (actionOutputData.exists()) { - int maxOutputData = getJobConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024); + int maxOutputData = getAppConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024); actionData.put(ACTION_DATA_OUTPUT_PROPS, getLocalFileContentStr(actionOutputData, "Output", maxOutputData)); } @@ -426,14 +422,14 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R private void uploadActionDataToHDFS() throws IOException { if (!actionData.isEmpty()) { Path finalPath = new Path(actionDir, ACTION_DATA_SEQUENCE_FILE); - FileSystem fs = FileSystem.get(finalPath.toUri(), getJobConf()); + FileSystem fs = FileSystem.get(finalPath.toUri(), getAppConf()); // upload into sequence file System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: " + new Path(actionDir, ACTION_DATA_SEQUENCE_FILE).toUri()); SequenceFile.Writer wr = null; try { - wr = SequenceFile.createWriter(fs, getJobConf(), finalPath, Text.class, Text.class); + wr = SequenceFile.createWriter(fs, getAppConf(), finalPath, Text.class, Text.class); if (wr != null) { for (Entry<String, String> entry : actionData.entrySet()) { wr.append(new Text(entry.getKey()), new Text(entry.getValue())); @@ -458,12 +454,12 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R private void setupMainConfiguration() throws IOException { Path pathNew = new Path(new Path(actionDir, ACTION_CONF_XML), new Path(new File(ACTION_CONF_XML).getAbsolutePath())); - FileSystem fs = FileSystem.get(pathNew.toUri(), getJobConf()); + FileSystem fs = FileSystem.get(pathNew.toUri(), getAppConf()); fs.copyToLocalFile(new Path(actionDir, ACTION_CONF_XML), new Path(new File(ACTION_CONF_XML).getAbsolutePath())); - System.setProperty("oozie.launcher.job.id", getJobConf().get("mapred.job.id")); - System.setProperty(OOZIE_JOB_ID, getJobConf().get(OOZIE_JOB_ID)); - System.setProperty(OOZIE_ACTION_ID, getJobConf().get(OOZIE_ACTION_ID)); + System.setProperty("oozie.launcher.job.id", getAppConf().get("mapred.job.id")); + System.setProperty(OOZIE_JOB_ID, getAppConf().get(OOZIE_JOB_ID)); + System.setProperty(OOZIE_ACTION_ID, getAppConf().get(OOZIE_ACTION_ID)); System.setProperty("oozie.action.conf.xml", new File(ACTION_CONF_XML).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS, new File(ACTION_DATA_EXTERNAL_CHILD_IDS).getAbsolutePath()); @@ -471,12 +467,12 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath()); - if (getJobConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME) != null) { + if (getAppConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME) != null) { System.setProperty(LauncherMain.OOZIE_JOB_LAUNCH_TIME, - getJobConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME)); + getAppConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME)); } - String actionConfigClass = getJobConf().get(OOZIE_ACTION_CONFIG_CLASS); + String actionConfigClass = getAppConf().get(OOZIE_ACTION_CONFIG_CLASS); if (actionConfigClass != null) { System.setProperty(OOZIE_ACTION_CONFIG_CLASS, actionConfigClass); } @@ -484,10 +480,10 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R // Method to execute the prepare actions private void executePrepare() throws IOException, LauncherException, ParserConfigurationException, SAXException { - String prepareXML = getJobConf().get(ACTION_PREPARE_XML); + String prepareXML = getAppConf().get(ACTION_PREPARE_XML); if (prepareXML != null) { if (!prepareXML.equals("")) { - Configuration actionConf = new Configuration(getJobConf()); + Configuration actionConf = new Configuration(getAppConf()); String actionXml = System.getProperty("oozie.action.conf.xml"); actionConf.addResource(new Path("file:///", actionXml)); PrepareActionsDriver.doOperations(prepareXML, actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java index 4e8bb4b..a844aa0 100644 --- a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java +++ b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java @@ -47,8 +47,6 @@ import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.WorkflowAppService; -import com.google.common.base.Preconditions; - public class OozieSharelibCLI { public static final String[] HELP_INFO = { "", @@ -170,7 +168,7 @@ public class OozieSharelibCLI { System.out.println("the destination path for sharelib is: " + dstPath); URI uri = new Path(hdfsUri).toUri(); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = FileSystem.get(uri, fsConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/5998c18f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java index 7fff802..ccad273 100644 --- a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java +++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java @@ -183,7 +183,7 @@ public class TestOozieSharelibCLI extends XTestCase { if (fs == null) { HadoopAccessorService has = getServices().get(HadoopAccessorService.class); URI uri = new Path(outPath).toUri(); - Configuration fsConf = has.createJobConf(uri.getAuthority()); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf); } return fs;
