Author: shv
Date: Tue Jun 30 18:48:45 2009
New Revision: 789875
URL: http://svn.apache.org/viewvc?rev=789875&view=rev
Log:
HADOOP-6080. Introduce -skipTrash option to rm and rmr. Contributed by Jakob
Homan.
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=789875&r1=789874&r2=789875&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue Jun 30 18:48:45 2009
@@ -154,6 +154,9 @@
HADOOP-3315. Add a new, binary file foramt, TFile. (Hong Tang via cdouglas)
+ HADOOP-6080. Introduce -skipTrash option to rm and rmr.
+ (Jakob Homan via shv)
+
IMPROVEMENTS
HADOOP-4565. Added CombineFileInputFormat to use data locality information
Modified:
hadoop/common/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml?rev=789875&r1=789874&r2=789875&view=diff
==============================================================================
--- hadoop/common/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
(original)
+++ hadoop/common/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
Tue Jun 30 18:48:45 2009
@@ -325,10 +325,13 @@
<section>
<title> rm </title>
<p>
- <code>Usage: hadoop fs -rm URI [URI …]
</code>
+ <code>Usage: hadoop fs -rm [-skipTrash] URI
[URI …] </code>
</p>
<p>
- Delete files specified as args. Only deletes non empty directory and
files. Refer to rmr for recursive deletes.<br/>
+ Delete files specified as args. Only deletes non empty directory and
files. If the <code>-skipTrash</code> option
+ is specified, the trash, if enabled, will be bypassed and the
specified file(s) deleted immediately. This can be
+ useful when it is necessary to delete files from an
over-quota directory.
+ Refer to rmr for recursive deletes.<br/>
Example:
</p>
<ul>
@@ -344,9 +347,11 @@
<section>
<title> rmr </title>
<p>
- <code>Usage: hadoop fs -rmr URI [URI
…]</code>
+ <code>Usage: hadoop fs -rmr [-skipTrash] URI
[URI …]</code>
</p>
- <p>Recursive version of delete.<br/>
+ <p>Recursive version of delete. If the
<code>-skipTrash</code> option
+ is specified, the trash, if enabled, will be bypassed and
the specified file(s) deleted immediately. This can be
+ useful when it is necessary to delete files from an
over-quota directory.<br/>
Example:
</p>
<ul>
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java?rev=789875&r1=789874&r2=789875&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java Tue Jun 30
18:48:45 2009
@@ -1086,10 +1086,12 @@
* Delete all files that match the file pattern <i>srcf</i>.
* @param srcf a file pattern specifying source files
* @param recursive if need to delete subdirs
+ * @param skipTrash Should we skip the trash, if it's enabled?
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
- void delete(String srcf, final boolean recursive) throws IOException {
+ void delete(String srcf, final boolean recursive, final boolean skipTrash)
+ throws IOException
{
//rm behavior in Linux
// [~/1207]$ ls ?.txt
// x.txt z.txt
@@ -1100,22 +1102,27 @@
new DelayedExceptionThrowing() {
@Override
void process(Path p, FileSystem srcFs) throws IOException {
- delete(p, srcFs, recursive);
+ delete(p, srcFs, recursive, skipTrash);
}
}.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
}
/* delete a file */
- private void delete(Path src, FileSystem srcFs, boolean recursive) throws
IOException {
+ private void delete(Path src, FileSystem srcFs, boolean recursive,
+ boolean skipTrash) throws IOException {
if (srcFs.isDirectory(src) && !recursive) {
throw new IOException("Cannot remove directory \"" + src +
"\", use -rmr instead");
}
- Trash trashTmp = new Trash(srcFs, getConf());
- if (trashTmp.moveToTrash(src)) {
- System.out.println("Moved to trash: " + src);
- return;
+
+ if(!skipTrash) {
+ Trash trashTmp = new Trash(srcFs, getConf());
+ if (trashTmp.moveToTrash(src)) {
+ System.out.println("Moved to trash: " + src);
+ return;
+ }
}
+
if (srcFs.delete(src, true)) {
System.out.println("Deleted " + src);
} else {
@@ -1312,8 +1319,8 @@
"The full syntax is: \n\n" +
"hadoop fs [-fs <local | file system URI>] [-conf <configuration
file>]\n\t" +
"[-D <property=value>] [-ls <path>] [-lsr <path>] [-df [<path>]] [-du
<path>]\n\t" +
- "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm <src>]\n\t" +
- "[-rmr <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ...
<dst>]\n\t" +
+ "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm [-skipTrash]
<src>]\n\t" +
+ "[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal
<localsrc> ... <dst>]\n\t" +
"[-moveFromLocal <localsrc> ... <dst>] [" +
GET_SHORT_USAGE + "\n\t" +
"[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
@@ -1381,11 +1388,15 @@
"\t\tdestination. When copying multiple files, the destination\n" +
"\t\tmust be a directory. \n";
- String rm = "-rm <src>: \tDelete all files that match the specified file
pattern.\n" +
- "\t\tEquivlent to the Unix command \"rm <src>\"\n";
-
- String rmr = "-rmr <src>: \tRemove all directories which match the
specified file \n" +
- "\t\tpattern. Equivlent to the Unix command \"rm -rf <src>\"\n";
+ String rm = "-rm [-skipTrash] <src>: \tDelete all files that match the
specified file pattern.\n" +
+ "\t\tEquivalent to the Unix command \"rm <src>\"\n" +
+ "\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" +
+ "deletes <src>";
+
+ String rmr = "-rmr [-skipTrash] <src>: \tRemove all directories which
match the specified file \n" +
+ "\t\tpattern. Equivalent to the Unix command \"rm -rf <src>\"\n" +
+ "\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" +
+ "deletes <src>";
String put = "-put <localsrc> ... <dst>: \tCopy files " +
"from the local file system \n\t\tinto fs. \n";
@@ -1573,8 +1584,6 @@
System.out.println(Count.DESCRIPTION);
System.out.println(help);
}
-
-
}
/**
@@ -1584,6 +1593,15 @@
private int doall(String cmd, String argv[], int startindex) {
int exitCode = 0;
int i = startindex;
+ boolean rmSkipTrash = false;
+
+ // Check for -skipTrash option in rm/rmr
+ if(("-rm".equals(cmd) || "-rmr".equals(cmd))
+ && "-skipTrash".equals(argv[i])) {
+ rmSkipTrash = true;
+ i++;
+ }
+
//
// for each source file, issue the command
//
@@ -1597,9 +1615,9 @@
} else if ("-mkdir".equals(cmd)) {
mkdir(argv[i]);
} else if ("-rm".equals(cmd)) {
- delete(argv[i], false);
+ delete(argv[i], false, rmSkipTrash);
} else if ("-rmr".equals(cmd)) {
- delete(argv[i], true);
+ delete(argv[i], true, rmSkipTrash);
} else if ("-df".equals(cmd)) {
df(argv[i]);
} else if (Count.matches(cmd)) {
@@ -1708,8 +1726,8 @@
System.err.println(" [" + Count.USAGE + "]");
System.err.println(" [-mv <src> <dst>]");
System.err.println(" [-cp <src> <dst>]");
- System.err.println(" [-rm <path>]");
- System.err.println(" [-rmr <path>]");
+ System.err.println(" [-rm [-skipTrash] <path>]");
+ System.err.println(" [-rmr [-skipTrash] <path>]");
System.err.println(" [-expunge]");
System.err.println(" [-put <localsrc> ... <dst>]");
System.err.println(" [-copyFromLocal <localsrc> ... <dst>]");
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java?rev=789875&r1=789874&r2=789875&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java
(original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java Tue
Jun 30 18:48:45 2009
@@ -140,7 +140,7 @@
// Verify that we can recreate the file
writeFile(fs, myFile);
-
+
// Verify that we succeed in removing the whole directory
// along with the file inside it.
{
@@ -256,6 +256,63 @@
assertTrue(val == -1);
assertTrue(fs.exists(trashRoot));
}
+
+ // Verify skip trash option really works
+
+ // recreate directory and file
+ mkdir(fs, myPath);
+ writeFile(fs, myFile);
+
+ // Verify that skip trash option really skips the trash for files (rm)
+ {
+ String[] args = new String[3];
+ args[0] = "-rm";
+ args[1] = "-skipTrash";
+ args[2] = myFile.toString();
+ int val = -1;
+ try {
+ // Clear out trash
+ assertEquals(0, shell.run(new String [] { "-expunge" } ));
+
+ val = shell.run(args);
+
+ }catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertFalse(fs.exists(trashRoot)); // No new Current should be created
+ assertFalse(fs.exists(myFile));
+ assertTrue(val == 0);
+ }
+
+ // recreate directory and file
+ mkdir(fs, myPath);
+ writeFile(fs, myFile);
+
+ // Verify that skip trash option really skips the trash for rmr
+ {
+ String[] args = new String[3];
+ args[0] = "-rmr";
+ args[1] = "-skipTrash";
+ args[2] = myPath.toString();
+
+ int val = -1;
+ try {
+ // Clear out trash
+ assertEquals(0, shell.run(new String [] { "-expunge" } ));
+
+ val = shell.run(args);
+
+ }catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+
+ assertFalse(fs.exists(trashRoot)); // No new Current should be created
+ assertFalse(fs.exists(myPath));
+ assertFalse(fs.exists(myFile));
+ assertTrue(val == 0);
+ }
}
public static void trashNonDefaultFS(Configuration conf) throws IOException {
