Merge branch 'apache_forked_master' into apache_forked_oya Change-Id: Id369937bb6ae485eeaf620c7002cd546b5b98471
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a1537e1b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a1537e1b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a1537e1b Branch: refs/heads/oya Commit: a1537e1bb7cf381b822740377c16f258b24a5453 Parents: 876aceb fe182fd Author: Peter Bacsko <[email protected]> Authored: Tue Dec 13 12:05:50 2016 +0100 Committer: Peter Bacsko <[email protected]> Committed: Tue Dec 13 12:05:50 2016 +0100 ---------------------------------------------------------------------- bin/mkdistro.sh | 2 +- bin/test-patch-11-findbugs-diff | 28 +- bin/test-patch-20-tests | 4 + .../java/org/apache/oozie/cli/OozieCLI.java | 22 +- core/pom.xml | 5 - core/src/main/conf/oozie-env.sh | 5 +- .../oozie/action/hadoop/JavaActionExecutor.java | 8 + .../apache/oozie/coord/CoordELFunctions.java | 7 +- .../jpa/WorkflowsJobGetJPAExecutor.java | 1 + .../apache/oozie/store/StoreStatusFilter.java | 1 + .../java/org/apache/oozie/util/ConfigUtils.java | 12 +- .../apache/oozie/util/ELConstantsFunctions.java | 6 +- .../org/apache/oozie/util/Instrumentation.java | 8 +- core/src/main/resources/oozie-default.xml | 93 +++- .../action/hadoop/TestJavaActionExecutor.java | 1 + .../client/TestOozieClientWithFakeServlet.java | 4 +- .../oozie/coord/TestCoordELFunctions.java | 36 ++ .../jpa/TestBundleJobInfoGetJPAExecutor.java | 19 + .../jpa/TestCoordJobInfoGetJPAExecutor.java | 19 + .../jpa/TestWorkflowsJobGetJPAExecutor.java | 19 + .../TestCoordMaterializeTriggerService.java | 55 +- .../java/org/apache/oozie/test/XTestCase.java | 2 + distro/pom.xml | 42 +- distro/src/main/bin/oozie-jetty-server.sh | 226 +++++++++ distro/src/main/bin/oozie-setup.sh | 182 ++----- distro/src/main/bin/oozie-sys.sh | 33 +- distro/src/main/bin/oozied.sh | 87 +--- distro/src/main/tomcat/ROOT/WEB-INF/web.xml | 25 - distro/src/main/tomcat/ROOT/favicon.ico | Bin 1150 -> 0 bytes distro/src/main/tomcat/ROOT/index.html | 25 - distro/src/main/tomcat/logging.properties | 64 --- distro/src/main/tomcat/server.xml | 150 ------ distro/src/main/tomcat/ssl-server.xml | 152 ------ distro/src/main/tomcat/ssl-web.xml | 295 ----------- docs/src/site/twiki/DG_QuickStart.twiki | 7 +- docs/src/site/twiki/ENG_Building.twiki | 3 +- pom.xml | 487 +++++++++++++++++- release-log.txt | 16 +- server/pom.xml | 257 ++++++++++ server/src/main/assemblies/empty.xml | 21 + .../oozie/server/EmbeddedOozieServer.java | 206 ++++++++ .../org/apache/oozie/server/FilterMapper.java | 61 +++ .../oozie/server/HttpConfigurationWrapper.java | 63 +++ .../org/apache/oozie/server/JspHandler.java | 158 ++++++ .../oozie/server/SSLServerConnectorFactory.java | 136 +++++ .../org/apache/oozie/server/ServletMapper.java | 95 ++++ .../oozie/server/WebRootResourceLocator.java | 39 ++ .../ConstraintSecurityHandlerProvider.java | 47 ++ .../oozie/server/guice/JettyServerProvider.java | 48 ++ .../oozie/server/guice/JspHandlerProvider.java | 47 ++ .../oozie/server/guice/OozieGuiceModule.java | 45 ++ .../server/guice/RewriteHandlerProvider.java | 44 ++ .../oozie/server/guice/ServicesProvider.java | 39 ++ server/src/main/resources/checkstyle-header.txt | 17 + server/src/main/resources/checkstyle.xml | 41 ++ .../oozie/server/TestEmbeddedOozieServer.java | 119 +++++ .../org/apache/oozie/server/TestJspHandler.java | 94 ++++ .../server/TestSSLServerConnectorFactory.java | 137 +++++ sharelib/hive/pom.xml | 34 ++ sharelib/pig/pom.xml | 34 ++ .../hadoop/TestMapReduceActionExecutor.java | 23 +- src/main/assemblies/distro.xml | 60 +-- webapp/src/main/webapp/403.html | 31 ++ webapp/src/main/webapp/404.html | 31 ++ webapp/src/main/webapp/index.jsp | 1 - webapp/src/main/webapp/json2.js | 498 ------------------- 66 files changed, 2987 insertions(+), 1590 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/core/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index b8c870c,4beeb96..371f171 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@@ -25,7 -25,7 +25,8 @@@ import java.net.ConnectException import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; +import java.nio.ByteBuffer; + import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; @@@ -46,14 -46,14 +47,18 @@@ import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AccessControlException; -import org.apache.oozie.hadoop.utils.HadoopShims; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskLog; +import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; +import org.apache.hadoop.mapreduce.v2.util.MRApps; + import org.apache.hadoop.mapred.JobID; + import org.apache.hadoop.mapred.RunningJob; + import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; + import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.DiskChecker; @@@ -85,8 -71,8 +90,9 @@@ import org.apache.oozie.service.HadoopA import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.URIHandlerService; + import org.apache.oozie.service.UserGroupInformationService; import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.util.ClasspathUtils; import org.apache.oozie.util.ELEvaluationException; import org.apache.oozie.util.ELEvaluator; import org.apache.oozie.util.JobUtils; @@@ -1515,18 -1593,41 +1521,20 @@@ public class JavaActionExecutor extend @Override public void kill(Context context, WorkflowAction action) throws ActionExecutorException { - JobClient jobClient = null; - boolean exception = false; + YarnClient yarnClient = null; try { Element actionXml = XmlUtils.parseXml(action.getConf()); - final JobConf jobConf = createBaseHadoopConf(context, actionXml); - WorkflowJob wfJob = context.getWorkflow(); - Configuration conf = null; - if ( wfJob.getConf() != null ) { - conf = new XConfiguration(new StringReader(wfJob.getConf())); - } - String launcherTag = LauncherMapperHelper.getActionYarnTag(conf, wfJob.getParentId(), action); - jobConf.set(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS, LauncherMapperHelper.getTag(launcherTag)); - jobConf.set(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME, Long.toString(action.getStartTime().getTime())); - UserGroupInformation ugi = Services.get().get(UserGroupInformationService.class) - .getProxyUser(context.getWorkflow().getUser()); - ugi.doAs(new PrivilegedExceptionAction<Void>() { - @Override - public Void run() throws Exception { - LauncherMainHadoopUtils.killChildYarnJobs(jobConf); - return null; - } - }); - jobClient = createJobClient(context, jobConf); - RunningJob runningJob = getRunningJob(context, action, jobClient); - if (runningJob != null) { - runningJob.killJob(); - } ++ + JobConf jobConf = createBaseHadoopConf(context, actionXml); + yarnClient = createYarnClient(context, jobConf); + yarnClient.killApplication(ConverterUtils.toApplicationId(action.getExternalId())); ++ context.setExternalStatus(KILLED); context.setExecutionData(KILLED, null); - } - catch (Exception ex) { - exception = true; + } catch (Exception ex) { + LOG.error("Error: ", ex); throw convertException(ex); - } - finally { + } finally { try { FileSystem actionFs = context.getAppFileSystem(); cleanUpActionDir(actionFs, context); http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index bfc8ab4,8965cdf..aa485ef --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@@ -489,7 -543,68 +489,8 @@@ public class TestJavaActionExecutor ext assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); } - public void testChildKill() throws Exception { - final JobConf clusterConf = createJobConf(); - FileSystem fileSystem = FileSystem.get(clusterConf); - Path confFile = new Path("/tmp/cluster-conf.xml"); - OutputStream out = fileSystem.create(confFile); - clusterConf.writeXml(out); - out.close(); - String confFileName = fileSystem.makeQualified(confFile).toString() + "#core-site.xml"; - final String actionXml = "<java>" + - "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + - "<name-node>" + getNameNodeUri() + "</name-node>" + - "<main-class> " + SleepJob.class.getName() + " </main-class>" + - "<arg>-mt</arg>" + - "<arg>300000</arg>" + - "<archive>" + confFileName + "</archive>" + - "</java>"; - final Context context = createContext(actionXml, null); - final RunningJob runningJob = submitAction(context); - waitFor(60 * 1000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.getJobStatus().getRunState() == 1; - } - }); - assertFalse(runningJob.isComplete()); - Thread.sleep(15000); - JavaActionExecutor ae = new JavaActionExecutor(); - ae.kill(context, context.getAction()); - - WorkflowJob wfJob = context.getWorkflow(); - Configuration conf = null; - if (wfJob.getConf() != null) { - conf = new XConfiguration(new StringReader(wfJob.getConf())); - } - String launcherTag = LauncherMapperHelper.getActionYarnTag(conf, wfJob.getParentId(), context.getAction()); - final Configuration jobConf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml)); - jobConf.set(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS, LauncherMapperHelper.getTag(launcherTag)); - jobConf.setLong(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME, context.getAction().getStartTime().getTime()); - - UserGroupInformationService ugiService = Services.get().get(UserGroupInformationService.class); - UserGroupInformation ugi = ugiService.getProxyUser(getTestUser()); - Set<String> childSet = ugi.doAs(new PrivilegedExceptionAction<Set<String>>() { - @Override - public Set<String> run() throws Exception { - Set<String> childSet = LauncherMainHadoopUtils.getChildJobs(jobConf); - return childSet; - } - }); - assertEquals(1, childSet.size()); - - JobClient jobClient = new JobClient(clusterConf); - for (String jobId : childSet) { - RunningJob childJob = jobClient.getJob(jobId); - assertEquals(JobStatus.State.KILLED.getValue(), childJob.getJobStatus().getRunState()); - } - assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); - assertEquals(WorkflowAction.Status.DONE, context.getAction().getStatus()); - assertEquals("KILLED", context.getAction().getExternalStatus()); - assertFalse(runningJob.isSuccessful()); - } + public void testExceptionSubmitException() throws Exception { + - public void testExceptionSubmitException() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/docs/src/site/twiki/DG_QuickStart.twiki ---------------------------------------------------------------------- diff --cc docs/src/site/twiki/DG_QuickStart.twiki index 88f0533,2b9dc3b..2454985 --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@@ -41,14 -41,18 +41,14 @@@ suitable when same oozie package needs 2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are available to customise the versions of the dependencies: --P<profile> - default hadoop-2. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop -profile depending on the hadoop version used. --Ptez - Bundle tez jars in hive and pig sharelibs. Useful if you want to use tez -+as the execution engine for those applications. --Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 +-Dhadoop.version=<version> - default 2.4.0 -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version --Dpig.version=<version> - default 0.16.0 --Dpig.classifier=<classifier> - default none +-Dpig.version=<version> - default 0.12.1 +-Dpig.classifier=<classifier> - default h2 -Dsqoop.version=<version> - default 1.4.3 - -Dsqoop.classifier=<classifier> - default hadoop200 - -Dtomcat.version=<version> - default 6.0.44 + -Dsqoop.classifier=<classifier> - default hadoop100 + -Dtomcat.version=<version> - default 6.0.47 -Dopenjpa.version=<version> - default 2.2.2 -Dxerces.version=<version> - default 2.10.0 -Dcurator.version=<version> - default 2.5.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/pom.xml ---------------------------------------------------------------------- diff --cc pom.xml index 7ea789d,0cb5c78..403b626 --- a/pom.xml +++ b/pom.xml @@@ -101,12 -100,14 +101,13 @@@ <spark.bagel.version>1.6.1</spark.bagel.version> <spark.guava.version>14.0.1</spark.guava.version> <spark.scala.binary.version>2.10</spark.scala.binary.version> - <sqoop.classifier>hadoop100</sqoop.classifier> + <sqoop.classifier>hadoop200</sqoop.classifier> <streaming.version>${hadoop.version}</streaming.version> - <distcp.version>${hadoop.version}</distcp.version> <hadoop.auth.version>${hadoop.version}</hadoop.auth.version> + <tez.version>0.8.4</tez.version> + + <jetty.version>9.2.19.v20160908</jetty.version> - <!-- Tomcat version --> - <tomcat.version>6.0.44</tomcat.version> <jline.version>0.9.94</jline.version> <openjpa.version>2.4.1</openjpa.version> <xerces.version>2.10.0</xerces.version> http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/release-log.txt ---------------------------------------------------------------------- diff --cc release-log.txt index e850a24,5aa5352..9660968 --- a/release-log.txt +++ b/release-log.txt @@@ -1,15 -1,26 +1,27 @@@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2590 OYA: Create basic Oozie Launcher Application Master (rkanter) +OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter) + OOZIE-2616 Add Tez profile for Hive and Pig sharelibs (poeppt via rkanter) + OOZIE-2741 Remove Tomcat (asasvari via rkanter) + OOZIE-2745 test-patch should also list the failed tests (gezapeti via rkanter) + OOZIE-2740 oozie help misspelled coordinator (coordiantor) and retrieved (retreived) (gsohn via rkanter) + OOZIE-2690 OOZIE NPE while executing kill() (abhishekbafna via jaydeepvishwakarma) + OOZIE-2737 testConfigDefaultPropsToAction is flaky (satishsaley via rohini) + OOZIE-2666 Support embedding Jetty into Oozie (asasvari via rkanter) OOZIE-1459 Remove the version in the child poms for maven-antrun-plugin (Jan Hentschel via rkanter) - OOZIE-2225 Add wild card filter for gathering jobs (sai-krish via rkanter) - OOZIE-2536 Hadoop's cleanup of local directory in uber mode causing failures (satishsaley via rohini) + OOZIE-2225 Add wild card filter for gathering jobs (sai-krish,pbacsko via rkanter,rohini) OOZIE-1986 Add FindBugs report to pre-commit build (andras.piros via rkanter) OOZIE-2634 Queue dump command message is confusing when the queue is empty (andras.piros via rkanter) - -- Oozie 4.3.0 release + OOZIE-2742 Unable to kill applications based on tag (satishsaley via rohini) + OOZIE-2720 Test failure - TestCoordMaterializeTriggerService#testMaxMatThrottleNotPicked (gezapeti via rohini) + OOZIE-2536 Hadoop's cleanup of local directory in uber mode causing failures (satishsaley via rohini) + OOZIE-2723 JSON.org license is now CatX (rkanter, abhishekbafna via shwethags) + OOZIE-2725 Upgrade Tomcat to 6.0.47 for the latest security fixes (rkanter via shwethags) + OOZIE-2724 coord:current resolves monthly/yearly dependencies incorrectly (satishsaley via shwethags) OOZIE-2719 Test case failure (abhishekbafna via jaydeepvishwakarma) OOZIE-2674 Improve oozie commads documentation (abhishekbafna via rkanter) OOZIE-2710 Oozie HCatalog example workflow fails (abhishekbafna via shwethags) http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/sharelib/hive/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/sharelib/pig/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1537e1b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java ---------------------------------------------------------------------- diff --cc sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java index 88f9a7c,c7860be..ac7af7c --- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java +++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java @@@ -39,39 -68,11 +39,40 @@@ import java.util.jar.JarOutputStream import java.util.regex.Pattern; import java.util.zip.ZipEntry; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.hadoop.streaming.StreamJob; +import org.apache.oozie.WorkflowActionBean; +import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.action.ActionExecutorException; +import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.WorkflowAction; ++import org.apache.oozie.client.WorkflowAction.Status; +import org.apache.oozie.command.wf.StartXCommand; +import org.apache.oozie.command.wf.SubmitXCommand; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor.WorkflowActionQuery; +import org.apache.oozie.service.HadoopAccessorService; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.util.ClassUtils; +import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.PropertiesUtils; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.XmlUtils; +import org.jdom.Element; public class TestMapReduceActionExecutor extends ActionExecutorTestCase {
