Author: cdouglas
Date: Fri Jan 23 14:49:24 2009
New Revision: 737219
URL: http://svn.apache.org/viewvc?rev=737219&view=rev
Log:
HADOOP-5086. Use the appropriate FileSystem for trash URIs.
Added:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestTrash.java
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/hdfs/TestHDFSTrash.java
Removed:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/hdfs/TestTrash.java
Modified:
hadoop/core/branches/branch-0.20/CHANGES.txt
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FsShell.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/Trash.java
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=737219&r1=737218&r2=737219&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Fri Jan 23 14:49:24 2009
@@ -625,6 +625,8 @@
HADOOP-5009. DataNode#shutdown sometimes leaves data block scanner
verification log unclosed. (hairong)
+ HADOOP-5086. Use the appropriate FileSystem for trash URIs. (cdouglas)
+
Release 0.19.0 - 2008-11-18
INCOMPATIBLE CHANGES
Modified:
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FsShell.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FsShell.java?rev=737219&r1=737218&r2=737219&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FsShell.java
(original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FsShell.java
Fri Jan 23 14:49:24 2009
@@ -1037,7 +1037,7 @@
throw new IOException("Cannot remove directory \"" + src +
"\", use -rmr instead");
}
- Trash trashTmp = new Trash(srcFs.getConf());
+ Trash trashTmp = new Trash(srcFs, getConf());
if (trashTmp.moveToTrash(src)) {
System.out.println("Moved to trash: " + src);
return;
Modified:
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/Trash.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/Trash.java?rev=737219&r1=737218&r2=737219&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/Trash.java
(original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/Trash.java
Fri Jan 23 14:49:24 2009
@@ -50,21 +50,31 @@
private static final DateFormat CHECKPOINT = new
SimpleDateFormat("yyMMddHHmm");
private static final int MSECS_PER_MINUTE = 60*1000;
- private FileSystem fs;
- private Path trash;
- private Path current;
- private long interval;
+ private final FileSystem fs;
+ private final Path trash;
+ private final Path current;
+ private final long interval;
/** Construct a trash can accessor.
* @param conf a Configuration
*/
public Trash(Configuration conf) throws IOException {
- this(FileSystem.get(conf).getHomeDirectory(), conf);
+ this(FileSystem.get(conf), conf);
}
- private Trash(Path home, Configuration conf) throws IOException {
+ /**
+ * Construct a trash can accessor for the FileSystem provided.
+ */
+ public Trash(FileSystem fs, Configuration conf) throws IOException {
super(conf);
+ this.fs = fs;
+ this.trash = new Path(fs.getHomeDirectory(), TRASH);
+ this.current = new Path(trash, CURRENT);
+ this.interval = conf.getLong("fs.trash.interval", 60) * MSECS_PER_MINUTE;
+ }
+ private Trash(Path home, Configuration conf) throws IOException {
+ super(conf);
this.fs = home.getFileSystem(conf);
this.trash = new Path(home, TRASH);
this.current = new Path(trash, CURRENT);
Added:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestTrash.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestTrash.java?rev=737219&view=auto
==============================================================================
---
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestTrash.java
(added)
+++
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestTrash.java
Fri Jan 23 14:49:24 2009
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+
+import junit.framework.TestCase;
+import java.io.File;
+import java.io.IOException;
+import java.io.DataOutputStream;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.fs.LocalFileSystem;
+
+/**
+ * This class tests commands from Trash.
+ */
+public class TestTrash extends TestCase {
+
+ private final static Path TEST_DIR =
+ new Path(new File(System.getProperty("test.build.data","/tmp")
+ ).toURI().toString().replace(' ', '+'), "testTrash");
+
+ protected static Path writeFile(FileSystem fs, Path f) throws IOException {
+ DataOutputStream out = fs.create(f);
+ out.writeBytes("dhruba: " + f);
+ out.close();
+ assertTrue(fs.exists(f));
+ return f;
+ }
+
+ protected static Path mkdir(FileSystem fs, Path p) throws IOException {
+ assertTrue(fs.mkdirs(p));
+ assertTrue(fs.exists(p));
+ assertTrue(fs.getFileStatus(p).isDir());
+ return p;
+ }
+
+ // check that the specified file is in Trash
+ protected static void checkTrash(FileSystem fs, Path trashRoot,
+ String pathname) throws IOException {
+ Path p = new Path(trashRoot+"/"+new Path(pathname).getName());
+ assertTrue(fs.exists(p));
+ }
+
+ // check that the specified file is not in Trash
+ static void checkNotInTrash(FileSystem fs, Path trashRoot, String pathname)
+ throws IOException {
+ Path p = new Path(trashRoot+"/"+ new Path(pathname).getName());
+ assertTrue(!fs.exists(p));
+ }
+
+ protected static void trashShell(final FileSystem fs, final Path base)
+ throws IOException {
+ Configuration conf = new Configuration();
+ conf.set("fs.trash.interval", "10"); // 10 minute
+ conf.set("fs.default.name", fs.getUri().toString());
+ FsShell shell = new FsShell();
+ shell.setConf(conf);
+ Path trashRoot = null;
+
+ // First create a new directory with mkdirs
+ Path myPath = new Path(base, "test/mkdirs");
+ mkdir(fs, myPath);
+
+ // Second, create a file in that directory.
+ Path myFile = new Path(base, "test/mkdirs/myFile");
+ writeFile(fs, myFile);
+
+ // Verify that expunge without Trash directory
+ // won't throw Exception
+ {
+ String[] args = new String[1];
+ args[0] = "-expunge";
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ }
+
+ // Verify that we succeed in removing the file we created.
+ // This should go into Trash.
+ {
+ String[] args = new String[2];
+ args[0] = "-rm";
+ args[1] = new Path(base, "test/mkdirs/myFile").toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+
+ trashRoot = shell.getCurrentTrashDir();
+ checkTrash(fs, trashRoot, args[1]);
+ }
+
+ // Verify that we can recreate the file
+ writeFile(fs, myFile);
+
+ // Verify that we succeed in removing the file we re-created
+ {
+ String[] args = new String[2];
+ args[0] = "-rm";
+ args[1] = new Path(base, "test/mkdirs/myFile").toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ }
+
+ // Verify that we can recreate the file
+ writeFile(fs, myFile);
+
+ // Verify that we succeed in removing the whole directory
+ // along with the file inside it.
+ {
+ String[] args = new String[2];
+ args[0] = "-rmr";
+ args[1] = new Path(base, "test/mkdirs").toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ }
+
+ // recreate directory
+ mkdir(fs, myPath);
+
+ // Verify that we succeed in removing the whole directory
+ {
+ String[] args = new String[2];
+ args[0] = "-rmr";
+ args[1] = new Path(base, "test/mkdirs").toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ }
+
+ // Check that we can delete a file from the trash
+ {
+ Path toErase = new Path(trashRoot, "toErase");
+ int retVal = -1;
+ writeFile(fs, toErase);
+ try {
+ retVal = shell.run(new String[] {"-rm", toErase.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(retVal == 0);
+ checkNotInTrash (fs, trashRoot, toErase.toString());
+ checkNotInTrash (fs, trashRoot, toErase.toString()+".1");
+ }
+
+ // simulate Trash removal
+ {
+ String[] args = new String[1];
+ args[0] = "-expunge";
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ }
+
+ // verify that after expunging the Trash, it really goes away
+ checkNotInTrash(fs, trashRoot, new Path(base,
"test/mkdirs/myFile").toString());
+
+ // recreate directory and file
+ mkdir(fs, myPath);
+ writeFile(fs, myFile);
+
+ // remove file first, then remove directory
+ {
+ String[] args = new String[2];
+ args[0] = "-rm";
+ args[1] = new Path(base, "test/mkdirs/myFile").toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ checkTrash(fs, trashRoot, args[1]);
+
+ args = new String[2];
+ args[0] = "-rmr";
+ args[1] = new Path(base, "test/mkdirs").toString();
+ val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+ checkTrash(fs, trashRoot, args[1]);
+ }
+
+ // attempt to remove parent of trash
+ {
+ String[] args = new String[2];
+ args[0] = "-rmr";
+ args[1] = trashRoot.getParent().getParent().toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == -1);
+ assertTrue(fs.exists(trashRoot));
+ }
+ }
+
+ public static void trashNonDefaultFS(Configuration conf) throws IOException {
+ conf.set("fs.trash.interval", "10"); // 10 minute
+ // attempt non-default FileSystem trash
+ {
+ final FileSystem lfs = FileSystem.getLocal(conf);
+ Path p = TEST_DIR;
+ Path f = new Path(p, "foo/bar");
+ if (lfs.exists(p)) {
+ lfs.delete(p, true);
+ }
+ try {
+ f = writeFile(lfs, f);
+
+ FileSystem.closeAll();
+ Trash lTrash =
+ new Trash(FileSystem.get(URI.create("file:///"), conf), conf);
+ lTrash.moveToTrash(f.getParent());
+ assertTrue(lfs.exists(new Path(lTrash.getCurrentTrashDir(),
"foo/bar")));
+ } finally {
+ if (lfs.exists(p)) {
+ lfs.delete(p, true);
+ }
+ }
+ }
+ }
+
+ public void testTrash() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+ trashShell(FileSystem.getLocal(conf), TEST_DIR);
+ }
+
+ public void testNonDefaultFS() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+ conf.set("fs.default.name", "invalid://host/bar/foo");
+ trashNonDefaultFS(conf);
+ }
+
+ static class TestLFS extends LocalFileSystem {
+ Path home;
+ TestLFS() {
+ this(TEST_DIR);
+ }
+ TestLFS(Path home) {
+ super();
+ this.home = home;
+ }
+ public Path getHomeDirectory() {
+ return home;
+ }
+ }
+}
Added:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/hdfs/TestHDFSTrash.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/hdfs/TestHDFSTrash.java?rev=737219&view=auto
==============================================================================
---
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/hdfs/TestHDFSTrash.java
(added)
+++
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/hdfs/TestHDFSTrash.java
Fri Jan 23 14:49:24 2009
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+import junit.extensions.TestSetup;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.TestTrash;
+
+/**
+ * This class tests commands from Trash.
+ */
+public class TestHDFSTrash extends TestTrash {
+
+ private static MiniDFSCluster cluster = null;
+ public static Test suite() {
+ TestSetup setup = new TestSetup(new TestSuite(TestHDFSTrash.class)) {
+ protected void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ cluster = new MiniDFSCluster(conf, 2, true, null);
+ }
+ protected void tearDown() throws Exception {
+ if (cluster != null) { cluster.shutdown(); }
+ }
+ };
+ return setup;
+ }
+
+ /**
+ * Tests Trash on HDFS
+ */
+ public void testTrash() throws IOException {
+ trashShell(cluster.getFileSystem(), new Path("/"));
+ }
+
+ public void testNonDefaultFS() throws IOException {
+ FileSystem fs = cluster.getFileSystem();
+ Configuration conf = fs.getConf();
+ conf.set("fs.default.name", fs.getUri().toString());
+ trashNonDefaultFS(conf);
+ }
+
+}
