Repository: hadoop Updated Branches: refs/heads/branch-2.8 352d5a7e8 -> d84d54e26
HDFS-9812. Streamer threads leak if failure happens when closing DFSOutputStream. Contributed by Lin Yiqun. (cherry picked from commit 352d299cf8ebe330d24117df98d1e6a64ae38c26) (cherry picked from commit fe0009a2bd17e26f9e9364ec3f772e9a3f138de8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d84d54e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d84d54e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d84d54e2 Branch: refs/heads/branch-2.8 Commit: d84d54e26ca1aceba33439d02c381a639e606973 Parents: 352d5a7 Author: Akira Ajisaka <[email protected]> Authored: Tue Mar 8 10:43:17 2016 +0900 Committer: Akira Ajisaka <[email protected]> Committed: Tue Mar 8 10:57:56 2016 +0900 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d84d54e2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 5878756..6d29ec8 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -774,14 +774,19 @@ public class DFSOutputStream extends FSOutputSummer flushInternal(); // flush all data to Datanodes // get last block before destroying the streamer ExtendedBlock lastBlock = getStreamer().getBlock(); - closeThreads(false); + try (TraceScope ignored = dfsClient.getTracer().newScope("completeFile")) { completeFile(lastBlock); } } catch (ClosedChannelException ignored) { } finally { - setClosed(); + // Failures may happen when flushing data. + // Streamers may keep waiting for the new block information. + // Thus need to force closing these threads. + // Don't need to call setClosed() because closeThreads(true) + // calls setClosed() in the finally block. + closeThreads(true); } }
