This is an automated email from the ASF dual-hosted git repository. xingtanzjr pushed a commit to branch fix_1.0.1_region_allocation in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1b8cd6e1e0579ec644506517b71f089d904a7564 Author: YongzaoDan <[email protected]> AuthorDate: Thu Jan 12 18:39:59 2023 +0800 Finish (#8849) --- .../iotdb/confignode/conf/ConfigNodeConfig.java | 4 +- .../confignode/manager/ClusterSchemaManager.java | 79 +++++++++++----------- .../manager/ClusterSchemaManagerTest.java | 38 +++++++++++ 3 files changed, 81 insertions(+), 40 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 94e906a9ad..4be8758fde 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -95,10 +95,10 @@ public class ConfigNodeConfig { private double dataRegionPerProcessor = 1.0; /** The least number of SchemaRegionGroup for each Database. */ - private int leastSchemaRegionGroupNum = 1; + private volatile int leastSchemaRegionGroupNum = 1; /** The least number of DataRegionGroup for each Database. */ - private int leastDataRegionGroupNum = 5; + private volatile int leastDataRegionGroupNum = 5; /** RegionGroup allocate policy. */ private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy = diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java index 861017f294..91746ace72 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java @@ -97,9 +97,7 @@ public class ClusterSchemaManager { private static final Logger LOGGER = LoggerFactory.getLogger(ClusterSchemaManager.class); private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); - private static final int LEAST_SCHEMA_REGION_GROUP_NUM = CONF.getLeastSchemaRegionGroupNum(); private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode(); - private static final int LEAST_DATA_REGION_GROUP_NUM = CONF.getLeastDataRegionGroupNum(); private static final double DATA_REGION_PER_PROCESSOR = CONF.getDataRegionPerProcessor(); private final IManager configManager; @@ -316,6 +314,11 @@ public class ClusterSchemaManager { // Get all StorageGroupSchemas Map<String, TStorageGroupSchema> storageGroupSchemaMap = getMatchedStorageGroupSchemasByName(getStorageGroupNames()); + if (storageGroupSchemaMap.size() == 0) { + // Skip when there are no StorageGroups + return; + } + int dataNodeNum = getNodeManager().getRegisteredDataNodeCount(); int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount(); int storageGroupNum = storageGroupSchemaMap.size(); @@ -348,24 +351,13 @@ public class ClusterSchemaManager { .getRegionGroupCount( storageGroupSchema.getName(), TConsensusGroupType.SchemaRegion); int maxSchemaRegionGroupNum = - Math.max( - // The least number of SchemaRegionGroup of each StorageGroup is specified - // by parameter least_schema_region_group_num, which is currently unconfigurable. - LEAST_SCHEMA_REGION_GROUP_NUM, - Math.max( - (int) - // Use Math.ceil here to ensure that the maxSchemaRegionGroupNum - // will be increased as long as the number of cluster DataNodes is increased - Math.ceil( - // The maxSchemaRegionGroupNum of the current StorageGroup - // is expected to be: - // (SCHEMA_REGION_PER_DATA_NODE * registerDataNodeNum) / - // (createdStorageGroupNum * schemaReplicationFactor) - SCHEMA_REGION_PER_DATA_NODE - * dataNodeNum - / storageGroupNum - * storageGroupSchema.getSchemaReplicationFactor()), - allocatedSchemaRegionGroupCount)); + calcMaxRegionGroupNum( + CONF.getLeastSchemaRegionGroupNum(), + SCHEMA_REGION_PER_DATA_NODE, + dataNodeNum, + storageGroupNum, + storageGroupSchema.getSchemaReplicationFactor(), + allocatedSchemaRegionGroupCount); LOGGER.info( "[AdjustRegionGroupNum] The maximum number of SchemaRegionGroups for Database: {} is adjusted to: {}", storageGroupSchema.getName(), @@ -378,24 +370,13 @@ public class ClusterSchemaManager { getPartitionManager() .getRegionGroupCount(storageGroupSchema.getName(), TConsensusGroupType.DataRegion); int maxDataRegionGroupNum = - Math.max( - // The least number of DataRegionGroup of each StorageGroup is specified - // by parameter least_data_region_group_num. - LEAST_DATA_REGION_GROUP_NUM, - Math.max( - (int) - // Use Math.ceil here to ensure that the maxDataRegionGroupNum - // will be increased as long as the number of cluster DataNodes is increased - Math.ceil( - // The maxDataRegionGroupNum of the current StorageGroup - // is expected to be: - // (DATA_REGION_PER_PROCESSOR * totalCpuCoreNum) / - // (createdStorageGroupNum * dataReplicationFactor) - DATA_REGION_PER_PROCESSOR - * totalCpuCoreNum - / storageGroupNum - * storageGroupSchema.getDataReplicationFactor()), - allocatedDataRegionGroupCount)); + calcMaxRegionGroupNum( + CONF.getLeastDataRegionGroupNum(), + DATA_REGION_PER_PROCESSOR, + totalCpuCoreNum, + storageGroupNum, + storageGroupSchema.getDataReplicationFactor(), + allocatedDataRegionGroupCount); LOGGER.info( "[AdjustRegionGroupNum] The maximum number of DataRegionGroups for Database: {} is adjusted to: {}", storageGroupSchema.getName(), @@ -411,6 +392,28 @@ public class ClusterSchemaManager { getConsensusManager().write(adjustMaxRegionGroupNumPlan); } + public static int calcMaxRegionGroupNum( + int leastRegionGroupNum, + double resourceWeight, + int resource, + int storageGroupNum, + int replicationFactor, + int allocatedRegionGroupCount) { + return Math.max( + // The maxRegionGroupNum should be great or equal to the leastRegionGroupNum + leastRegionGroupNum, + Math.max( + (int) + // Use Math.ceil here to ensure that the maxRegionGroupNum + // will be increased as long as the number of cluster DataNodes is increased + Math.ceil( + // The maxRegionGroupNum of the current StorageGroup is expected to be: + // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor) + resourceWeight * resource / (double) (storageGroupNum * replicationFactor)), + // The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount + allocatedRegionGroupCount)); + } + // ====================================================== // Leader scheduling interfaces // ====================================================== diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java new file mode 100644 index 0000000000..2af715eb1b --- /dev/null +++ b/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.manager; + +import org.junit.Assert; +import org.junit.Test; + +public class ClusterSchemaManagerTest { + + @Test + public void testCalcMaxRegionGroupNum() { + + // The maxRegionGroupNum should be great or equal to the leastRegionGroupNum + Assert.assertEquals(100, ClusterSchemaManager.calcMaxRegionGroupNum(100, 1.0, 3, 1, 3, 0)); + + // The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount + Assert.assertEquals(100, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 6, 2, 3, 100)); + + // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor) + Assert.assertEquals(20, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 120, 2, 3, 5)); + } +}
