Repository: hadoop Updated Branches: refs/heads/HDFS-7240 830474fe3 -> 9a3159cb6
HDFS-12423. Ozone: TopN container choosing policy should ignore containers that has no pending deletion blocks. Contributed by Yiqun Lin. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a3159cb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a3159cb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a3159cb Branch: refs/heads/HDFS-7240 Commit: 9a3159cb6a25df3b5eb66517ac9b8f028c4a81fb Parents: 830474f Author: Weiwei Yang <[email protected]> Authored: Wed Sep 13 10:50:57 2017 +0800 Committer: Weiwei Yang <[email protected]> Committed: Wed Sep 13 10:50:57 2017 +0800 ---------------------------------------------------------------------- ...pNOrderedContainerDeletionChoosingPolicy.java | 19 +++++++++++++------ .../background/BlockDeletingService.java | 3 +++ .../common/TestBlockDeletingService.java | 5 +++++ .../TestContainerDeletionChoosingPolicy.java | 12 ++++++++++-- 4 files changed, 31 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java index b867179..3f4cdaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java @@ -66,13 +66,20 @@ public class TopNOrderedContainerDeletionChoosingPolicy int currentCount = 0; for (ContainerStatus entry : orderedList) { if (currentCount < count) { - result.add(entry.getContainer()); - currentCount++; + if (entry.getNumPendingDeletionBlocks() > 0) { + result.add(entry.getContainer()); + currentCount++; - LOG.debug("Select container {} for block deletion, " - + "pending deletion blocks num: {}.", - entry.getContainer().getContainerName(), - entry.getNumPendingDeletionBlocks()); + LOG.debug( + "Select container {} for block deletion, " + + "pending deletion blocks num: {}.", + entry.getContainer().getContainerName(), + entry.getNumPendingDeletionBlocks()); + } else { + LOG.debug("Stop looking for next container, there is no" + + " pending deletion block contained in remaining containers."); + break; + } } else { break; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java index 1a2507b..644e928 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java @@ -106,6 +106,9 @@ public class BlockDeletingService extends BackgroundService{ // configured. containers = containerManager.chooseContainerForBlockDeletion( containerLimitPerInterval); + LOG.info("Plan to choose {} containers for block deletion, " + + "actually returns {} valid containers.", + containerLimitPerInterval, containers.size()); for(ContainerData container : containers) { BlockDeletingTask containerTask = http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index ceeb659..7ff5294 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -31,9 +31,11 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerData; import org.apache.hadoop.ozone.container.common.helpers.KeyData; import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl; +import org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy; import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager; import org.apache.hadoop.ozone.container.common.statemachine.background.BlockDeletingService; import org.apache.hadoop.ozone.web.utils.OzoneUtils; +import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.utils.BackgroundService; @@ -99,6 +101,9 @@ public class TestBlockDeletingService { private ContainerManager createContainerManager(Configuration conf) throws Exception { + // use random container choosing policy for testing + conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY, + RandomContainerDeletionChoosingPolicy.class.getName()); conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, containersDir.getAbsolutePath()); if (containersDir.exists()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java index a1c91cc..d6ef9d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java @@ -138,7 +138,8 @@ public class TestContainerDeletionChoosingPolicy { int numContainers = 10; Random random = new Random(); Map<String, Integer> name2Count = new HashMap<>(); - for (int i = 0; i < numContainers; i++) { + // create [numContainers + 1] containers + for (int i = 0; i <= numContainers; i++) { String containerName = OzoneUtils.getRequestID(); ContainerData data = new ContainerData(containerName); containerManager.createContainer(createSingleNodePipeline(containerName), @@ -146,6 +147,11 @@ public class TestContainerDeletionChoosingPolicy { Assert.assertTrue( containerManager.getContainerMap().containsKey(containerName)); + // don't create deletion blocks in the last container. + if (i == numContainers) { + break; + } + // create random number of deletion blocks and write to container db int deletionBlocks = random.nextInt(numContainers) + 1; // record <ContainerName, DeletionCount> value @@ -170,7 +176,9 @@ public class TestContainerDeletionChoosingPolicy { Assert.assertEquals(5, result0.size()); List<ContainerData> result1 = containerManager - .chooseContainerForBlockDeletion(numContainers); + .chooseContainerForBlockDeletion(numContainers + 1); + // the empty deletion blocks container should not be chosen + Assert.assertEquals(numContainers, result1.size()); // verify the order of return list int lastCount = Integer.MAX_VALUE; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
