Repository: hadoop Updated Branches: refs/heads/branch-2 fed2339e2 -> fe0009a2b
HDFS-9812. Streamer threads leak if failure happens when closing DFSOutputStream. Contributed by Lin Yiqun. (cherry picked from commit 352d299cf8ebe330d24117df98d1e6a64ae38c26) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe0009a2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe0009a2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe0009a2 Branch: refs/heads/branch-2 Commit: fe0009a2bd17e26f9e9364ec3f772e9a3f138de8 Parents: fed2339 Author: Akira Ajisaka <[email protected]> Authored: Tue Mar 8 10:43:17 2016 +0900 Committer: Akira Ajisaka <[email protected]> Committed: Tue Mar 8 10:55:59 2016 +0900 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe0009a2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 5878756..6d29ec8 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -774,14 +774,19 @@ public class DFSOutputStream extends FSOutputSummer flushInternal(); // flush all data to Datanodes // get last block before destroying the streamer ExtendedBlock lastBlock = getStreamer().getBlock(); - closeThreads(false); + try (TraceScope ignored = dfsClient.getTracer().newScope("completeFile")) { completeFile(lastBlock); } } catch (ClosedChannelException ignored) { } finally { - setClosed(); + // Failures may happen when flushing data. + // Streamers may keep waiting for the new block information. + // Thus need to force closing these threads. + // Don't need to call setClosed() because closeThreads(true) + // calls setClosed() in the finally block. + closeThreads(true); } }
