Repository: oozie Updated Branches: refs/heads/master 3fb549f3a -> f7f7628b8
OOZIE-2139 Coord update doesn't work for job which is submitted by bundle Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f7f7628b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f7f7628b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f7f7628b Branch: refs/heads/master Commit: f7f7628b8c7a681461958541fd247c77169aef13 Parents: 3fb549f Author: Purshotam Shah <[email protected]> Authored: Wed Apr 8 16:30:44 2015 -0700 Committer: Purshotam Shah <[email protected]> Committed: Wed Apr 8 16:30:44 2015 -0700 ---------------------------------------------------------------------- .../command/coord/CoordUpdateXCommand.java | 23 +++- .../apache/oozie/servlet/BaseJobServlet.java | 9 ++ .../command/coord/TestCoordUpdateXCommand.java | 109 ++++++++++++++++++- core/src/test/resources/bundle-submit-job.xml | 4 + release-log.txt | 1 + 5 files changed, 141 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f7f7628b/core/src/main/java/org/apache/oozie/command/coord/CoordUpdateXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordUpdateXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordUpdateXCommand.java index 6647ef8..92ad4f4 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordUpdateXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordUpdateXCommand.java @@ -23,6 +23,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import java.util.Date; + +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.ErrorCode; @@ -54,6 +56,9 @@ public class CoordUpdateXCommand extends CoordSubmitXCommand { private boolean showDiff = true; private boolean isConfChange = false; + //This properties are set in coord jobs by bundle. An update command should not overide it. + final static String[] bundleConfList = new String[] { OozieClient.BUNDLE_ID }; + StringBuffer diff = new StringBuffer(); CoordinatorJobBean oldCoordJob = null; @@ -116,9 +121,23 @@ public class CoordUpdateXCommand extends CoordSubmitXCommand { } LogUtils.setLogInfo(oldCoordJob); - if (!isConfChange) { + if (!isConfChange || StringUtils.isEmpty(conf.get(OozieClient.COORDINATOR_APP_PATH))) { try { - conf = new XConfiguration(new StringReader(oldCoordJob.getConf())); + XConfiguration jobConf = new XConfiguration(new StringReader(oldCoordJob.getConf())); + + if (!isConfChange) { + conf = jobConf; + } + else { + for (String bundleConfKey : bundleConfList) { + if (jobConf.get(bundleConfKey) != null) { + conf.set(bundleConfKey, jobConf.get(bundleConfKey)); + } + } + if (StringUtils.isEmpty(conf.get(OozieClient.COORDINATOR_APP_PATH))) { + conf.set(OozieClient.COORDINATOR_APP_PATH, jobConf.get(OozieClient.COORDINATOR_APP_PATH)); + } + } } catch (Exception e) { throw new CommandException(ErrorCode.E1023, e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/oozie/blob/f7f7628b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java index a581f8b..bd9e998 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java @@ -170,8 +170,17 @@ public abstract class BaseJobServlet extends JsonRestServlet { conf.set(OozieClient.USER_NAME, requestUser); } if (conf.get(OozieClient.COORDINATOR_APP_PATH) != null) { + //If coord is submitted from bundle, user may want to update individual coord job with bundle properties + //If COORDINATOR_APP_PATH is set, we should check only COORDINATOR_APP_PATH path permission + String bundlePath = conf.get(OozieClient.BUNDLE_APP_PATH); + if (bundlePath != null) { + conf.unset(OozieClient.BUNDLE_APP_PATH); + } BaseJobServlet.checkAuthorizationForApp(conf); JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf); + if (bundlePath != null) { + conf.set(OozieClient.BUNDLE_APP_PATH, bundlePath); + } } JSONObject json = updateJob(request, response, conf); startCron(); http://git-wip-us.apache.org/repos/asf/oozie/blob/f7f7628b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java index 89b81d5..de16d92 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java @@ -23,18 +23,29 @@ import java.io.FileWriter; import java.io.IOException; import java.io.Reader; import java.io.StringReader; +import java.io.UnsupportedEncodingException; import java.io.Writer; +import java.util.Date; +import java.util.List; +import java.util.regex.Matcher; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.BundleActionBean; +import org.apache.oozie.BundleJobBean; import org.apache.oozie.CoordinatorJobBean; -import org.apache.oozie.ErrorCode; import org.apache.oozie.XException; import org.apache.oozie.client.CoordinatorAction; +import org.apache.oozie.client.Job; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; +import org.apache.oozie.command.bundle.BundleStartXCommand; +import org.apache.oozie.command.bundle.BundleSubmitXCommand; +import org.apache.oozie.executor.jpa.BundleActionQueryExecutor; import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor; import org.apache.oozie.executor.jpa.JPAExecutorException; +import org.apache.oozie.executor.jpa.BundleActionQueryExecutor.BundleActionQuery; import org.apache.oozie.local.LocalOozie; import org.apache.oozie.service.JPAService; import org.apache.oozie.service.Services; @@ -278,13 +289,105 @@ public class TestCoordUpdateXCommand extends XDataTestCase { .getChildren("instance", namespace).get(0)).getText(); assertEquals(text, "${coord:future(0, 1)}"); new CoordActionsKillXCommand(jobId, RestConstants.JOB_COORD_SCOPE_ACTION, Integer.toString(actionNum)).call(); - coordClient.reRunCoord(jobId, RestConstants.JOB_COORD_SCOPE_ACTION, Integer.toString(actionNum), true, - true); + coordClient.reRunCoord(jobId, RestConstants.JOB_COORD_SCOPE_ACTION, Integer.toString(actionNum), true, true); bean = coordClient.getCoordActionInfo(actionId); sleep(1000); assertEquals(bean.getMissingDependencies(), "!!${coord:future(0, 1)}"); } + public void testCoordFromBundleJobChangeConf() throws Exception { + final XConfiguration jobConf = new XConfiguration(); + String coordJobId = setUpBundleAndGetCoordID(jobConf); + + jobConf.set("newvalue", "yes"); + CoordUpdateXCommand update = new CoordUpdateXCommand(false, jobConf, coordJobId); + update.call(); + CoordinatorJobBean job = getCoordJobs(coordJobId); + Configuration xConf = new XConfiguration(new StringReader(job.getConf())); + assertEquals(xConf.get("newvalue"), "yes"); + /* + * testProperty is part of bundle.xml <property> <name>testProperty</name> <value>abc</value> </property> + */ + jobConf.set("testProperty", "xyz"); + new CoordUpdateXCommand(false, jobConf, coordJobId).call(); + job = getCoordJobs(coordJobId); + xConf = new XConfiguration(new StringReader(job.getConf())); + assertEquals(xConf.get("testProperty"), "xyz"); + + } + + public void testCoordFromBundleJobChangeDefinition() throws Exception { + final XConfiguration jobConf = new XConfiguration(); + String coordJobId = setUpBundleAndGetCoordID(jobConf); + + CoordinatorJobBean job = getCoordJobs(coordJobId); + Element processedJobXml = XmlUtils.parseXml(job.getJobXml()); + Namespace namespace = processedJobXml.getNamespace(); + String text = ((Element) processedJobXml.getChild("input-events", namespace).getChild("data-in", namespace) + .getChildren("instance", namespace).get(0)).getText(); + assertEquals(text, "${coord:latest(0)}"); + + final Path coordPath1 = new Path(getFsTestCaseDir(), "coord1"); + writeCoordXml(coordPath1, "coord-multiple-input-instance4.xml"); + Configuration newConf = new Configuration(); + newConf.set(OozieClient.USER_NAME, getTestUser()); + CoordUpdateXCommand update = new CoordUpdateXCommand(false, newConf, coordJobId); + update.call(); + job = getCoordJobs(coordJobId); + processedJobXml = XmlUtils.parseXml(job.getJobXml()); + namespace = processedJobXml.getNamespace(); + text = ((Element) processedJobXml.getChild("input-events", namespace).getChild("data-in", namespace) + .getChildren("instance", namespace).get(0)).getText(); + assertEquals(text, "${coord:future(0, 1)}"); + } + + private String setUpBundleAndGetCoordID(XConfiguration jobConf) throws UnsupportedEncodingException, IOException, + CommandException, JPAExecutorException { + + final Path coordPath1 = new Path(getFsTestCaseDir(), "coord1"); + final Path coordPath2 = new Path(getFsTestCaseDir(), "coord2"); + writeCoordXml(coordPath1, "coord-multiple-input-instance3.xml"); + writeCoordXml(coordPath2, "coord-multiple-input-instance3.xml"); + + Path bundleAppPath = new Path(getFsTestCaseDir(), "bundle"); + String bundleAppXml = getBundleXml("bundle-submit-job.xml"); + assertNotNull(bundleAppXml); + assertTrue(bundleAppXml.length() > 0); + + bundleAppXml = bundleAppXml.replaceAll("#app_path1", + Matcher.quoteReplacement(new Path(coordPath1.toString(), "coordinator.xml").toString())); + bundleAppXml = bundleAppXml.replaceAll("#app_path2", + Matcher.quoteReplacement(new Path(coordPath2.toString(), "coordinator.xml").toString())); + + writeToFile(bundleAppXml, bundleAppPath, "bundle.xml"); + final Path appPath = new Path(bundleAppPath, "bundle.xml"); + jobConf.set(OozieClient.BUNDLE_APP_PATH, appPath.toString()); + jobConf.set("appName", "test"); + + jobConf.set(OozieClient.USER_NAME, getTestUser()); + + jobConf.set("coordName1", "NAME"); + jobConf.set("coordName2", "coord2"); + BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf); + final BundleJobBean bundleBean = (BundleJobBean) command.getJob(); + bundleBean.setStartTime(new Date()); + bundleBean.setEndTime(new Date()); + final String jobId = command.call(); + sleep(2000); + new BundleStartXCommand(jobId).call(); + waitFor(200000, new Predicate() { + public boolean evaluate() throws Exception { + List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList( + BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId); + return actions.get(0).getStatus().equals(Job.Status.RUNNING); + } + }); + + final List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList( + BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId); + return actions.get(0).getCoordId(); + } + private CoordinatorJobBean getCoordJobs(String jobId) { try { JPAService jpaService = Services.get().get(JPAService.class); http://git-wip-us.apache.org/repos/asf/oozie/blob/f7f7628b/core/src/test/resources/bundle-submit-job.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/bundle-submit-job.xml b/core/src/test/resources/bundle-submit-job.xml index a9599ef..6eda402 100644 --- a/core/src/test/resources/bundle-submit-job.xml +++ b/core/src/test/resources/bundle-submit-job.xml @@ -34,6 +34,10 @@ <name>END_TIME</name> <value>2009-02-20T23:59Z</value> </property> + <property> + <name>testProperty</name> + <value>abc</value> + </property> </configuration> </coordinator> <coordinator name='${coordName2}' critical='false'> http://git-wip-us.apache.org/repos/asf/oozie/blob/f7f7628b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index ae581ac..c7e0900 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2139 Coord update doesn't work for job which is submitted by bundle (puru) OOZIE-1726 Oozie does not support _HOST when configuring kerberos security (venkatnrangan via bzhang) OOZIE-2197 ooziedb.cmd command failed due to classpath being too long on windows (me.venkatr via bzhang) OOZIE-2182 SLA alert commands are not in sync with doc (puru)
