[37/50] hadoop git commit: HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Contributed by Rakesh R)
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/018893e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/018893e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/018893e8
Branch: refs/heads/YARN-2928
Commit: 018893e81ec1c43e6c79c77adec92c2edfb20cab
Parents: e537047
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 15:32:34 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 15:32:34 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 32 +---
.../apache/hadoop/hdfs/TestFileCreation.java| 4 +--
.../snapshot/TestRenameWithSnapshots.java | 4 +--
4 files changed, 35 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ad3e880..bbe1f02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -327,6 +327,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
+(Rakesh R via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 9208ed2..a6cc71f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -60,6 +60,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -118,6 +119,7 @@ import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* This class creates a single-process DFS cluster for junit testing.
@@ -523,7 +525,8 @@ public class MiniDFSCluster {
private boolean federation;
private boolean checkExitOnShutdown = true;
protected final int storagesPerDatanode;
-
+ private SetFileSystem fileSystems = Sets.newHashSet();
+
/**
* A unique instance identifier for the cluster. This
* is used to disambiguate HA filesystems in the case where
@@ -1705,6 +1708,13 @@ public class MiniDFSCluster {
* Shutdown all the nodes in the cluster.
*/
public void shutdown(boolean deleteDfsDir) {
+shutdown(deleteDfsDir, true);
+ }
+
+ /**
+ * Shutdown all the nodes in the cluster.
+ */
+ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
LOG.info(Shutting down the Mini HDFS Cluster);
if (checkExitOnShutdown) {
if (ExitUtil.terminateCalled()) {
@@ -1714,6 +1724,16 @@ public class MiniDFSCluster {
throw new AssertionError(Test resulted in an unexpected exit);
}
}
+if (closeFileSystem) {
+ for (FileSystem fs : fileSystems) {
+try {
+ fs.close();
+} catch (IOException ioe) {
+ LOG.warn(Exception while closing file system, ioe);
+}
+ }
+ fileSystems.clear();
+}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
if (nnInfo == null) continue;
@@ -2144,8 +2164,10 @@ public class MiniDFSCluster {
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
-return (DistributedFileSystem)FileSystem.get(getURI(nnIndex),
-nameNodes[nnIndex].conf);
+DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+getURI(nnIndex), nameNodes[nnIndex].conf);
+fileSystems.add(dfs);
+return dfs;
}
/**
@@ -2153,7 +2175,9 @@ public class MiniDFSCluster {
* This simulating different threads working on different FileSystem
instances.
*/
public FileSystem getNewFileSystemInstance(int nnIndex) throws IOException {
-return
hadoop git commit: HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Contributed by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c58786794 - 6ddb1bc85
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Contributed by Rakesh R)
(cherry picked from commit 018893e81ec1c43e6c79c77adec92c2edfb20cab)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ddb1bc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ddb1bc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ddb1bc8
Branch: refs/heads/branch-2
Commit: 6ddb1bc857b2ab85748171d1084882569f760c69
Parents: c587867
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 15:32:34 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 15:34:48 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 32 +---
.../apache/hadoop/hdfs/TestFileCreation.java| 4 +--
.../snapshot/TestRenameWithSnapshots.java | 4 +--
4 files changed, 35 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ddb1bc8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8e1a696..f9d2d32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -14,6 +14,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
+(Rakesh R via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ddb1bc8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 1076938..2113268 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -61,6 +61,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -120,6 +121,7 @@ import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* This class creates a single-process DFS cluster for junit testing.
@@ -525,7 +527,8 @@ public class MiniDFSCluster {
private boolean federation;
private boolean checkExitOnShutdown = true;
protected final int storagesPerDatanode;
-
+ private SetFileSystem fileSystems = Sets.newHashSet();
+
/**
* A unique instance identifier for the cluster. This
* is used to disambiguate HA filesystems in the case where
@@ -1709,6 +1712,13 @@ public class MiniDFSCluster {
* Shutdown all the nodes in the cluster.
*/
public void shutdown(boolean deleteDfsDir) {
+shutdown(deleteDfsDir, true);
+ }
+
+ /**
+ * Shutdown all the nodes in the cluster.
+ */
+ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
LOG.info(Shutting down the Mini HDFS Cluster);
if (checkExitOnShutdown) {
if (ExitUtil.terminateCalled()) {
@@ -1718,6 +1728,16 @@ public class MiniDFSCluster {
throw new AssertionError(Test resulted in an unexpected exit);
}
}
+if (closeFileSystem) {
+ for (FileSystem fs : fileSystems) {
+try {
+ fs.close();
+} catch (IOException ioe) {
+ LOG.warn(Exception while closing file system, ioe);
+}
+ }
+ fileSystems.clear();
+}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
if (nnInfo == null) continue;
@@ -2138,8 +2158,10 @@ public class MiniDFSCluster {
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
-return (DistributedFileSystem)FileSystem.get(getURI(nnIndex),
-nameNodes[nnIndex].conf);
+DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+getURI(nnIndex), nameNodes[nnIndex].conf);
+fileSystems.add(dfs);
+return dfs;
}
/**
@@ -2147,7 +2169,9 @@ public class MiniDFSCluster {
* This simulating different threads working on
hadoop git commit: HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Contributed by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/trunk e5370477c - 018893e81
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/018893e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/018893e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/018893e8
Branch: refs/heads/trunk
Commit: 018893e81ec1c43e6c79c77adec92c2edfb20cab
Parents: e537047
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 15:32:34 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 15:32:34 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 32 +---
.../apache/hadoop/hdfs/TestFileCreation.java| 4 +--
.../snapshot/TestRenameWithSnapshots.java | 4 +--
4 files changed, 35 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ad3e880..bbe1f02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -327,6 +327,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
+(Rakesh R via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 9208ed2..a6cc71f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -60,6 +60,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -118,6 +119,7 @@ import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* This class creates a single-process DFS cluster for junit testing.
@@ -523,7 +525,8 @@ public class MiniDFSCluster {
private boolean federation;
private boolean checkExitOnShutdown = true;
protected final int storagesPerDatanode;
-
+ private SetFileSystem fileSystems = Sets.newHashSet();
+
/**
* A unique instance identifier for the cluster. This
* is used to disambiguate HA filesystems in the case where
@@ -1705,6 +1708,13 @@ public class MiniDFSCluster {
* Shutdown all the nodes in the cluster.
*/
public void shutdown(boolean deleteDfsDir) {
+shutdown(deleteDfsDir, true);
+ }
+
+ /**
+ * Shutdown all the nodes in the cluster.
+ */
+ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
LOG.info(Shutting down the Mini HDFS Cluster);
if (checkExitOnShutdown) {
if (ExitUtil.terminateCalled()) {
@@ -1714,6 +1724,16 @@ public class MiniDFSCluster {
throw new AssertionError(Test resulted in an unexpected exit);
}
}
+if (closeFileSystem) {
+ for (FileSystem fs : fileSystems) {
+try {
+ fs.close();
+} catch (IOException ioe) {
+ LOG.warn(Exception while closing file system, ioe);
+}
+ }
+ fileSystems.clear();
+}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
if (nnInfo == null) continue;
@@ -2144,8 +2164,10 @@ public class MiniDFSCluster {
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
-return (DistributedFileSystem)FileSystem.get(getURI(nnIndex),
-nameNodes[nnIndex].conf);
+DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+getURI(nnIndex), nameNodes[nnIndex].conf);
+fileSystems.add(dfs);
+return dfs;
}
/**
@@ -2153,7 +2175,9 @@ public class MiniDFSCluster {
* This simulating different threads working on different FileSystem
instances.
*/
public FileSystem
[42/50] [abbrv] hadoop git commit: HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Contributed by Rakesh R)
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/018893e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/018893e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/018893e8
Branch: refs/heads/HDFS-7836
Commit: 018893e81ec1c43e6c79c77adec92c2edfb20cab
Parents: e537047
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 15:32:34 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 15:32:34 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 32 +---
.../apache/hadoop/hdfs/TestFileCreation.java| 4 +--
.../snapshot/TestRenameWithSnapshots.java | 4 +--
4 files changed, 35 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ad3e880..bbe1f02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -327,6 +327,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
+(Rakesh R via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 9208ed2..a6cc71f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -60,6 +60,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -118,6 +119,7 @@ import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* This class creates a single-process DFS cluster for junit testing.
@@ -523,7 +525,8 @@ public class MiniDFSCluster {
private boolean federation;
private boolean checkExitOnShutdown = true;
protected final int storagesPerDatanode;
-
+ private SetFileSystem fileSystems = Sets.newHashSet();
+
/**
* A unique instance identifier for the cluster. This
* is used to disambiguate HA filesystems in the case where
@@ -1705,6 +1708,13 @@ public class MiniDFSCluster {
* Shutdown all the nodes in the cluster.
*/
public void shutdown(boolean deleteDfsDir) {
+shutdown(deleteDfsDir, true);
+ }
+
+ /**
+ * Shutdown all the nodes in the cluster.
+ */
+ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
LOG.info(Shutting down the Mini HDFS Cluster);
if (checkExitOnShutdown) {
if (ExitUtil.terminateCalled()) {
@@ -1714,6 +1724,16 @@ public class MiniDFSCluster {
throw new AssertionError(Test resulted in an unexpected exit);
}
}
+if (closeFileSystem) {
+ for (FileSystem fs : fileSystems) {
+try {
+ fs.close();
+} catch (IOException ioe) {
+ LOG.warn(Exception while closing file system, ioe);
+}
+ }
+ fileSystems.clear();
+}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
if (nnInfo == null) continue;
@@ -2144,8 +2164,10 @@ public class MiniDFSCluster {
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
-return (DistributedFileSystem)FileSystem.get(getURI(nnIndex),
-nameNodes[nnIndex].conf);
+DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+getURI(nnIndex), nameNodes[nnIndex].conf);
+fileSystems.add(dfs);
+return dfs;
}
/**
@@ -2153,7 +2175,9 @@ public class MiniDFSCluster {
* This simulating different threads working on different FileSystem
instances.
*/
public FileSystem getNewFileSystemInstance(int nnIndex) throws IOException {
-return
