Repository: oozie Updated Branches: refs/heads/master 65271d99b -> cb6fb2fbf
OOZIE-2132 FS actions are not retried (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ced12712 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ced12712 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ced12712 Branch: refs/heads/master Commit: ced12712e60f265f4d639c82e86789d12f12d06b Parents: 65271d9 Author: egashira <[email protected]> Authored: Thu Mar 5 10:44:17 2015 -0800 Committer: egashira <[email protected]> Committed: Thu Mar 5 10:44:17 2015 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/FsActionExecutor.java | 10 ++++ core/src/main/resources/oozie-default.xml | 3 +- .../action/hadoop/TestFsActionExecutor.java | 35 +++++++++++ .../command/wf/TestActionStartXCommand.java | 63 ++++++++++++++++++++ release-log.txt | 1 + 5 files changed, 111 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/ced12712/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java index fed1d7a..013846f 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.AccessControlException; import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; @@ -55,6 +56,15 @@ public class FsActionExecutor extends ActionExecutor { maxGlobCount = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX); } + /** + * Initialize Action. + */ + @Override + public void initActionType() { + super.initActionType(); + registerError(AccessControlException.class.getName(), ActionExecutorException.ErrorType.ERROR, "FS014"); + } + Path getPath(Element element, String attribute) { String str = element.getAttributeValue(attribute).trim(); return new Path(str); http://git-wip-us.apache.org/repos/asf/oozie/blob/ced12712/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 27ef11d..9843330 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1930,10 +1930,11 @@ <property> <name>oozie.service.LiteWorkflowStoreService.user.retry.error.code</name> - <value>JA008,JA009,JA017,JA018,JA019,FS009,FS008</value> + <value>JA008,JA009,JA017,JA018,JA019,FS009,FS008,FS014</value> <description> Automatic retry interval for workflow action is handled for these specified error code: FS009, FS008 is file exists error when using chmod in fs action. + FS014 is permission error in fs action JA018 is output directory exists error in workflow map-reduce action. JA019 is error while executing distcp action. JA017 is job not exists error in action executor. http://git-wip-us.apache.org/repos/asf/oozie/blob/ced12712/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java index 350dacf..4ac6ffe 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java @@ -1092,4 +1092,39 @@ public void testChmodRecursive() throws Exception { fs.delete(basePath, true); } + + private Context createContextWithUser(String actionXml, String user) throws Exception { + FsActionExecutor ae = new FsActionExecutor(); + + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, user); + + WorkflowJobBean wf = createBaseWorkflow(protoConf, "fs-action"); + wf.setUser(user); + WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + action.setConf(actionXml); + + return new Context(wf, action); + } + + public void testRetryOnAccessControlError() throws Exception { + FsActionExecutor ae = new FsActionExecutor(); + FileSystem fs = getFileSystem(); + + Path path = new Path(getFsTestCaseDir(), "dir"); + fs.mkdirs(path); + fs.setPermission(path, FsPermission.valueOf("-rwx------")); + // do FS operation using different user to cause permission error + Context context = createContextWithUser("<fs/>", getTestUser2()); + try { + ae.chmod(context, path, "-r--------", false, false); + fail(); + } + catch (Exception e) { + assertTrue(e instanceof ActionExecutorException); + assertEquals("FS014", ((ActionExecutorException) e).getErrorCode()); + assertEquals(ActionExecutorException.ErrorType.ERROR, ((ActionExecutorException) e).getErrorType()); + } + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/ced12712/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java index b3f973a..b7489e9 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; @@ -44,7 +45,11 @@ import org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext; import org.apache.oozie.executor.jpa.JPAExecutorException; import org.apache.oozie.executor.jpa.WorkflowActionGetJPAExecutor; import org.apache.oozie.executor.jpa.WorkflowActionInsertJPAExecutor; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor.WorkflowActionQuery; import org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor; +import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor; +import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.InstrumentationService; import org.apache.oozie.service.JPAService; @@ -176,6 +181,19 @@ public class TestActionStartXCommand extends XDataTestCase { assertTrue(LauncherMapperHelper.hasIdSwap(actionData)); } + public void testActionStartToCheckRetry() throws Exception { + WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); + job.setUser(getTestUser2()); + WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW, job); + WorkflowActionBean action = this.addRecordToWfActionTableWithFS(job.getId(), "1", WorkflowAction.Status.PREP); + assertEquals(0, action.getUserRetryCount()); + assertEquals(1, action.getUserRetryMax()); + new ActionStartXCommand(action.getId(), "fs").call(); + action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, action.getId()); + assertNotNull(action.getExternalId()); + assertEquals(1, action.getUserRetryCount()); + } + public void testActionReuseWfJobAppPath() throws Exception { JPAService jpaService = Services.get().get(JPAService.class); WorkflowJobBean job = this.addRecordToWfJobTableWithCustomAppPath(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); @@ -373,6 +391,23 @@ public class TestActionStartXCommand extends XDataTestCase { return action; } + private WorkflowActionBean addRecordToWfActionTableWithFS(String wfId, String actionName, + WorkflowAction.Status status) throws Exception { + WorkflowActionBean action = createWorkflowActionForFS(wfId, status); + try { + JPAService jpaService = Services.get().get(JPAService.class); + assertNotNull(jpaService); + WorkflowActionInsertJPAExecutor actionInsertCmd = new WorkflowActionInsertJPAExecutor(action); + jpaService.execute(actionInsertCmd); + } + catch (JPAExecutorException ce) { + ce.printStackTrace(); + fail("Unable to insert the test wf action record to table"); + throw ce; + } + return action; + } + /** * Create workflow action with pending true * @@ -529,5 +564,33 @@ public class TestActionStartXCommand extends XDataTestCase { return action; } + private WorkflowActionBean createWorkflowActionForFS(String wfId, WorkflowAction.Status status) throws Exception { + WorkflowActionBean action = new WorkflowActionBean(); + String actionname = "testAction"; + action.setName(actionname); + action.setCred("null"); + action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); + action.setJobId(wfId); + action.setType("fs"); + action.setTransition("transition"); + action.setStatus(status); + action.setStartTime(new Date()); + action.setEndTime(new Date()); + action.setLastCheckTime(new Date()); + action.setPending(); + action.setExecutionPath("a"); + action.setRetries(1); + action.setUserRetryCount(0); + action.setUserRetryMax(1); + + Path path = new Path(getFsTestCaseDir(), "test"); + FileSystem fs = getFileSystem(); + fs.mkdirs(path); + fs.setPermission(path, FsPermission.valueOf("-rwx------")); + String actionXml = "<fs>" + "<chmod path='" + path.toString() + + "' permissions='-r--------' dir-files='false' />" + "</fs>"; + action.setConf(actionXml); + return action; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/ced12712/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b1c109a..3f10813 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2132 FS actions are not retried (ryota) OOZIE-2147 Oozie upgrade from 4.0.0 to 4.1.0 fails with CLOBs of zero length with Oracle DB (venkatnrangan via bzhang) OOZIE-2158 Overrides in action conf in streaming action do not work (rohini) OOZIE-2042 Max substitution for config variables should be configurable (seoeun25 via puru)
