Author: harsh
Date: Sun Sep 23 10:50:44 2012
New Revision: 1389010
URL: http://svn.apache.org/viewvc?rev=1389010&view=rev
Log:
HADOOP-8158. Interrupting hadoop fs -put from the command line causes a
LeaseExpiredException. Contributed by Daryn Sharp. (harsh)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1389010&r1=1389009&r2=1389010&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sun Sep 23
10:50:44 2012
@@ -219,6 +219,9 @@ Trunk (Unreleased)
HDFS-3678. Edit log files are never being purged from 2NN. (atm)
+ HADOOP-8158. Interrupting hadoop fs -put from the command line
+ causes a LeaseExpiredException. (daryn via harsh)
+
Release 2.0.3-alpha - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1389010&r1=1389009&r2=1389010&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
Sun Sep 23 10:50:44 2012
@@ -77,6 +77,7 @@ import javax.net.SocketFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStorageLocation;
@@ -696,6 +697,17 @@ public class DFSClient implements java.i
}
/**
+ * Close all open streams, abandoning all of the leases and files being
+ * created.
+ * @param abort whether streams should be gracefully closed
+ */
+ public void closeOutputStreams(boolean abort) {
+ if (clientRunning) {
+ closeAllFilesBeingWritten(abort);
+ }
+ }
+
+ /**
* Get the default block size for this cluster
* @return the default block size in bytes
*/
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1389010&r1=1389009&r2=1389010&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
Sun Sep 23 10:50:44 2012
@@ -535,10 +535,10 @@ public class DistributedFileSystem exten
@Override
public void close() throws IOException {
try {
- super.processDeleteOnExit();
- dfs.close();
- } finally {
+ dfs.closeOutputStreams(false);
super.close();
+ } finally {
+ dfs.close();
}
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1389010&r1=1389009&r2=1389010&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
Sun Sep 23 10:50:44 2012
@@ -22,7 +22,9 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@@ -54,6 +56,7 @@ import org.apache.hadoop.util.DataChecks
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.Test;
+import org.mockito.InOrder;
public class TestDistributedFileSystem {
private static final Random RAN = new Random();
@@ -127,7 +130,31 @@ public class TestDistributedFileSystem {
if (cluster != null) {cluster.shutdown();}
}
}
+
+ @Test
+ public void testDFSCloseOrdering() throws Exception {
+ DistributedFileSystem fs = new MyDistributedFileSystem();
+ Path path = new Path("/a");
+ fs.deleteOnExit(path);
+ fs.close();
+
+ InOrder inOrder = inOrder(fs.dfs);
+ inOrder.verify(fs.dfs).closeOutputStreams(eq(false));
+ inOrder.verify(fs.dfs).delete(eq(path.toString()), eq(true));
+ inOrder.verify(fs.dfs).close();
+ }
+ private static class MyDistributedFileSystem extends DistributedFileSystem {
+ MyDistributedFileSystem() {
+ statistics = new FileSystem.Statistics("myhdfs"); // can't mock finals
+ dfs = mock(DFSClient.class);
+ }
+ @Override
+ public boolean exists(Path p) {
+ return true; // trick out deleteOnExit
+ }
+ }
+
@Test
public void testDFSSeekExceptions() throws IOException {
Configuration conf = getTestConfiguration();
