HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by liaoyuxiangqin and Zsolt Venczel.
(cherry picked from commit 9efb4b7db00d79aded52997ec89a1be94ecdd268) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde45994 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde45994 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde45994 Branch: refs/remotes/origin/branch-3.1 Commit: fde45994a3598856934d17dfcf04eef27d95b73f Parents: ea2fa52 Author: Xiao Chen <x...@apache.org> Authored: Sun Jun 3 22:29:52 2018 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Sun Jun 3 22:37:27 2018 -0700 ---------------------------------------------------------------------- .../server/datanode/checker/DatasetVolumeChecker.java | 13 ++++++++++++- .../datanode/checker/TestDatasetVolumeChecker.java | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde45994/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 6ab6425..3889e23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -46,6 +46,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -103,6 +104,8 @@ public class DatasetVolumeChecker { private static final VolumeCheckContext IGNORED_CONTEXT = new VolumeCheckContext(); + private final ExecutorService checkVolumeResultHandlerExecutorService; + /** * @param conf Configuration object. * @param timer {@link Timer} object used for throttling checks. @@ -163,6 +166,12 @@ public class DatasetVolumeChecker { .setNameFormat("DataNode DiskChecker thread %d") .setDaemon(true) .build())); + + checkVolumeResultHandlerExecutorService = Executors.newCachedThreadPool( + new ThreadFactoryBuilder() + .setNameFormat("VolumeCheck ResultHandler thread %d") + .setDaemon(true) + .build()); } /** @@ -292,7 +301,9 @@ public class DatasetVolumeChecker { numVolumeChecks.incrementAndGet(); Futures.addCallback(olf.get(), new ResultHandler(volumeReference, new HashSet<>(), new HashSet<>(), - new AtomicLong(1), callback)); + new AtomicLong(1), callback), + checkVolumeResultHandlerExecutorService + ); return true; } else { IOUtils.cleanup(null, volumeReference); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde45994/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java index b37cc75..b0314f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.VolumeCheckContext; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.FakeTimer; import org.junit.Rule; @@ -122,6 +123,8 @@ public class TestDatasetVolumeChecker { } }); + GenericTestUtils.waitFor(() -> numCallbackInvocations.get() > 0, 5, 10000); + // Ensure that the check was invoked at least once. verify(volume, times(1)).check(anyObject()); if (result) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
