Repository: oozie Updated Branches: refs/heads/master f713d6bd7 -> 6be29c61f
OOZIE-2367 fs delete should support skipTrash option (jaydeepvishwakarma via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/6be29c61 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/6be29c61 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/6be29c61 Branch: refs/heads/master Commit: 6be29c61f11a9f6226bd2b771a2902ea4eed4668 Parents: f713d6b Author: Rohini Palaniswamy <[email protected]> Authored: Thu Oct 29 14:30:23 2015 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Thu Oct 29 14:30:23 2015 -0700 ---------------------------------------------------------------------- .../src/main/resources/oozie-workflow-0.5.xsd | 3 +- .../oozie/action/hadoop/FsActionExecutor.java | 40 ++++++++++++++++---- .../action/hadoop/TestFsActionExecutor.java | 22 ++++++++++- .../src/site/twiki/WorkflowFunctionalSpec.twiki | 7 +++- release-log.txt | 1 + 5 files changed, 62 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/6be29c61/client/src/main/resources/oozie-workflow-0.5.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/oozie-workflow-0.5.xsd b/client/src/main/resources/oozie-workflow-0.5.xsd index b01580c..fda49ed 100644 --- a/client/src/main/resources/oozie-workflow-0.5.xsd +++ b/client/src/main/resources/oozie-workflow-0.5.xsd @@ -282,7 +282,8 @@ </xs:complexType> <xs:complexType name="DELETE"> - <xs:attribute name="path" type="xs:string" use="required"/> + <xs:attribute name="path" type="xs:string" use="required" /> + <xs:attribute name="skip-trash" type="xs:boolean" use="optional" /> </xs:complexType> <xs:complexType name="MKDIR"> http://git-wip-us.apache.org/repos/asf/oozie/blob/6be29c61/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java index 8d96a47..b8649ac 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java @@ -21,6 +21,7 @@ package org.apache.oozie.action.hadoop; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -31,11 +32,14 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.ActionExecutorException; +import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.dependency.FSURIHandler; import org.apache.oozie.dependency.URIHandler; @@ -43,6 +47,7 @@ import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; +import org.apache.oozie.service.UserGroupInformationService; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; @@ -178,7 +183,12 @@ public class FsActionExecutor extends ActionExecutor { else { if (command.equals("delete")) { Path path = getPath(commandElement, "path"); - delete(context, fsConf, nameNodePath, path); + boolean skipTrash = true; + if (commandElement.getAttributeValue("skip-trash") != null && + commandElement.getAttributeValue("skip-trash").equals("false")) { + skipTrash = false; + } + delete(context, fsConf, nameNodePath, path, skipTrash); } else { if (command.equals("move")) { @@ -315,7 +325,6 @@ public class FsActionExecutor extends ActionExecutor { /** * @param path * @param user - * @param group * @return FileSystem * @throws HadoopAccessorException */ @@ -354,7 +363,7 @@ public class FsActionExecutor extends ActionExecutor { * @throws ActionExecutorException */ public void delete(Context context, Path path) throws ActionExecutorException { - delete(context, null, null, path); + delete(context, null, null, path, true); } /** @@ -366,7 +375,8 @@ public class FsActionExecutor extends ActionExecutor { * @param path * @throws ActionExecutorException */ - public void delete(Context context, XConfiguration fsConf, Path nameNodePath, Path path) throws ActionExecutorException { + public void delete(Context context, XConfiguration fsConf, Path nameNodePath, Path path, boolean skipTrash) + throws ActionExecutorException { URI uri = path.toUri(); URIHandler handler; try { @@ -374,13 +384,29 @@ public class FsActionExecutor extends ActionExecutor { if (handler instanceof FSURIHandler) { // Use legacy code to handle hdfs partition deletion path = resolveToFullPath(nameNodePath, path, true); - FileSystem fs = getFileSystemFor(path, context, fsConf); + final FileSystem fs = getFileSystemFor(path, context, fsConf); Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(path)); if (pathArr != null && pathArr.length > 0) { checkGlobMax(pathArr); - for (Path p : pathArr) { + for (final Path p : pathArr) { if (fs.exists(p)) { - if (!fs.delete(p, true)) { + if (!skipTrash) { + // Moving directory/file to trash of user. + UserGroupInformationService ugiService = Services.get().get(UserGroupInformationService.class); + UserGroupInformation ugi = ugiService.getProxyUser(fs.getConf().get(OozieClient.USER_NAME)); + ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { + @Override + public FileSystem run() throws Exception { + Trash trash = new Trash(fs.getConf()); + if (!trash.moveToTrash(p)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005", + "Could not move path [{0}] to trash on delete", p); + } + return null; + } + }); + } + else if (!fs.delete(p, true)) { throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005", "delete, path [{0}] could not delete path", p); } http://git-wip-us.apache.org/repos/asf/oozie/blob/6be29c61/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java index 86d5fa9..5345ae6 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java @@ -65,6 +65,7 @@ public class TestFsActionExecutor extends ActionExecutorTestCase { XConfiguration protoConf = new XConfiguration(); protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + protoConf.setLong("fs.trash.interval", 6000L); WorkflowJobBean wf = createBaseWorkflow(protoConf, "fs-action"); WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); @@ -257,6 +258,25 @@ public class TestFsActionExecutor extends ActionExecutorTestCase { } + public void testMovetoTrash() throws Exception { + FsActionExecutor ae = new FsActionExecutor(); + FileSystem fs = getFileSystem(); + + Path path = new Path(getFsTestCaseDir(), "dir1"); + + Context context = createContext("<fs/>"); + + fs.mkdirs(path); + + ae.delete(context, null, null, path, false); + + Path trashPath = new Path(fs.getHomeDirectory() + "/.Trash/Current/" + path.toUri().getPath() ); + + assertTrue(!fs.exists(path)); + + assertTrue(fs.exists(trashPath)); + } + @Override public void setUp() throws Exception { super.setUp(); @@ -293,7 +313,7 @@ public class TestFsActionExecutor extends ActionExecutorTestCase { Context context = createContext("<fs/>"); XConfiguration conf = new XConfiguration(); assertTrue(handler.exists(hcatURI, conf, getTestUser())); - ae.delete(context, conf, nameNodePath, path); + ae.delete(context, conf, nameNodePath, path, true); assertFalse(handler.exists(hcatURI, conf, getTestUser())); } http://git-wip-us.apache.org/repos/asf/oozie/blob/6be29c61/docs/src/site/twiki/WorkflowFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki index bf4d123..a084c1a 100644 --- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki +++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki @@ -1158,7 +1158,7 @@ executed. Thus there is less chance of an error occurring while the =fs= action ... <action name="[NODE-NAME]"> <fs> - <delete path='[PATH]'/> + <delete path='[PATH]' skip-trash='[true/false]'/> ... <mkdir path='[PATH]'/> ... @@ -1178,7 +1178,9 @@ executed. Thus there is less chance of an error occurring while the =fs= action </verbatim> The =delete= command deletes the specified path, if it is a directory it deletes recursively all its content and then -deletes the directory. It can also be used to drop hcat partitions. This is the only FS command which supports HCatalog URIs as well. For eg: <verbatim><delete path='hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value];...'/></verbatim> +deletes the directory. By default it does skip trash. It can be moved to trash by setting the value of skip-trash to +'false'. It can also be used to drop hcat partitions. This is the only FS command which supports HCatalog URIs as well. +For eg: <verbatim><delete path='hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value];...'/></verbatim> The =mkdir= command creates the specified directory, it creates all missing directories in the path. If the directory already exist it does a no-op. @@ -2868,6 +2870,7 @@ to be executed. <xs:complexType name="DELETE"> <xs:attribute name="path" type="xs:string" use="required"/> + <xs:attribute name="skip-trash" type="xs:boolean" use="optional"/> </xs:complexType> <xs:complexType name="MKDIR"> http://git-wip-us.apache.org/repos/asf/oozie/blob/6be29c61/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 9a0b9c5..ee46b29 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2367 fs delete should support skipTrash option (jaydeepvishwakarma via rohini) OOZIE-2368 coord:dateOffset and coord:dateTzOffset can't be used for coord initial-instance (puru) OOZIE-2369 coord:offset doesn't resolve correctly (puru) OOZIE-2251 Expose instrumental matrices in Realtime Graphing tool (nperiwal via rkanter)
