This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch add_multdir_strategy in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit e882ef35bcfd23b2a473a3caa99c2f64d9f2101e Author: mdf369 <[email protected]> AuthorDate: Fri Nov 22 11:19:00 2019 +0800 add RandomOnDiskUsableSpaceStrategy with UT --- .../resources/conf/iotdb-engine.properties | 1 + .../strategy/RandomOnDiskUsableSpaceStrategy.java | 63 ++++++++++++++++++++++ .../strategy/DirectoryStrategyTest.java | 23 +++++++- 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 9ef203a..db56592 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -109,6 +109,7 @@ timestamp_precision=ms # 1. SequenceStrategy: the system will choose the directory in sequence. # 2. MaxDiskUsableSpaceFirstStrategy: the system will choose the directory whose disk has the maximum space. # 3. MinFolderOccupiedSpaceFirstStrategy: the system will choose the directory whose folder has the minimum occupied space. +# 4. RandomOnDiskUsableSpaceStrategy: the system will randomly choose the directory based on usable space of disks. The more usable space, the greater the chance of being chosen; # Set SequenceStrategy,MaxDiskUsableSpaceFirstStrategy and MinFolderOccupiedSpaceFirstStrategy to apply the corresponding strategy. # If this property is unset, system will use MaxDiskUsableSpaceFirstStrategy as default strategy. # For this property, fully-qualified class name (include package name) and simple class name are both acceptable. diff --git a/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/RandomOnDiskUsableSpaceStrategy.java b/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/RandomOnDiskUsableSpaceStrategy.java new file mode 100644 index 0000000..411eb8a1 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/RandomOnDiskUsableSpaceStrategy.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.conf.directories.strategy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.utils.CommonUtils; + +public class RandomOnDiskUsableSpaceStrategy extends DirectoryStrategy { + + private Random random = new Random(System.currentTimeMillis()); + + @Override + public int nextFolderIndex() throws DiskSpaceInsufficientException { + List<Long> spaceList = getFolderUsableSpaceList(); + long spaceSum = spaceList.stream().mapToLong(Long::longValue).sum(); + + if (spaceSum <= 0) { + throw new DiskSpaceInsufficientException(folders); + } + + long randomV = Math.abs(random.nextLong()) % spaceSum; + int index = 0; + /* In fact, index will never equals spaceList.size(), + for that randomV is less than sum of spaceList. */ + while (index < spaceList.size() && randomV >= spaceList.get(index)) { + randomV -= spaceList.get(index); + index++; + } + + return index; + } + + /** + * get space list of all folders. + */ + public List<Long> getFolderUsableSpaceList() { + List<Long> spaceList = new ArrayList<>(); + for (int i = 0; i < folders.size(); i++) { + String folder = folders.get(i); + spaceList.add(CommonUtils.getUsableSpace(folder)); + } + return spaceList; + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java b/server/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java index 7d92328..8b61c6b 100644 --- a/server/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java +++ b/server/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.conf.directories.strategy; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -46,7 +48,7 @@ public class DirectoryStrategyTest { Set<Integer> fullDirIndexSet; @Before - public void setUp() throws DiskSpaceInsufficientException, IOException { + public void setUp() throws IOException { dataDirList = new ArrayList<>(); for (int i = 0; i < 5; i++) { dataDirList.add("data" + i); @@ -148,6 +150,25 @@ public class DirectoryStrategyTest { } @Test + public void testRandomOnDiskUsableSpaceStrategy() + throws DiskSpaceInsufficientException { + RandomOnDiskUsableSpaceStrategy randomOnDiskUsableSpaceStrategy = new RandomOnDiskUsableSpaceStrategy(); + randomOnDiskUsableSpaceStrategy.setFolders(dataDirList); + + for (int i = 0; i < dataDirList.size(); i++) { + assertFalse(fullDirIndexSet.contains(randomOnDiskUsableSpaceStrategy.nextFolderIndex())); + } + + int newFullIndex = randomOnDiskUsableSpaceStrategy.nextFolderIndex(); + PowerMockito.when(CommonUtils.getUsableSpace(dataDirList.get(newFullIndex))).thenReturn(0L); + for (int i = 0; i < dataDirList.size(); i++) { + int index = randomOnDiskUsableSpaceStrategy.nextFolderIndex(); + assertFalse(fullDirIndexSet.contains(index)); + assertTrue(newFullIndex != index); + } + } + + @Test public void testAllDiskFull() { for (int i = 0; i < dataDirList.size(); i++) { PowerMockito.when(CommonUtils.hasSpace(dataDirList.get(i))).thenReturn(false);
