This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a7f910ffc4 Finish (#8847)
a7f910ffc4 is described below
commit a7f910ffc4a332d9ce301fd3474237eff33f7e3b
Author: YongzaoDan <[email protected]>
AuthorDate: Thu Jan 12 18:39:31 2023 +0800
Finish (#8847)
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 4 +-
.../confignode/manager/ClusterSchemaManager.java | 79 +++++++++++-----------
.../manager/ClusterSchemaManagerTest.java | 38 +++++++++++
3 files changed, 81 insertions(+), 40 deletions(-)
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 1b48ba80b1..552b052ba4 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -96,10 +96,10 @@ public class ConfigNodeConfig {
private double dataRegionPerProcessor = 1.0;
/** The least number of SchemaRegionGroup for each Database. */
- private int leastSchemaRegionGroupNum = 1;
+ private volatile int leastSchemaRegionGroupNum = 1;
/** The least number of DataRegionGroup for each Database. */
- private int leastDataRegionGroupNum = 5;
+ private volatile int leastDataRegionGroupNum = 5;
/** RegionGroup allocate policy. */
private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
index 70341b82c2..6259a753f4 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
@@ -100,9 +100,7 @@ public class ClusterSchemaManager {
private static final Logger LOGGER =
LoggerFactory.getLogger(ClusterSchemaManager.class);
private static final ConfigNodeConfig CONF =
ConfigNodeDescriptor.getInstance().getConf();
- private static final int LEAST_SCHEMA_REGION_GROUP_NUM =
CONF.getLeastSchemaRegionGroupNum();
private static final double SCHEMA_REGION_PER_DATA_NODE =
CONF.getSchemaRegionPerDataNode();
- private static final int LEAST_DATA_REGION_GROUP_NUM =
CONF.getLeastDataRegionGroupNum();
private static final double DATA_REGION_PER_PROCESSOR =
CONF.getDataRegionPerProcessor();
private final IManager configManager;
@@ -319,6 +317,11 @@ public class ClusterSchemaManager {
// Get all StorageGroupSchemas
Map<String, TStorageGroupSchema> storageGroupSchemaMap =
getMatchedStorageGroupSchemasByName(getStorageGroupNames());
+ if (storageGroupSchemaMap.size() == 0) {
+ // Skip when there are no StorageGroups
+ return;
+ }
+
int dataNodeNum = getNodeManager().getRegisteredDataNodeCount();
int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount();
int storageGroupNum = storageGroupSchemaMap.size();
@@ -351,24 +354,13 @@ public class ClusterSchemaManager {
.getRegionGroupCount(
storageGroupSchema.getName(),
TConsensusGroupType.SchemaRegion);
int maxSchemaRegionGroupNum =
- Math.max(
- // The least number of SchemaRegionGroup of each StorageGroup
is specified
- // by parameter least_schema_region_group_num, which is
currently unconfigurable.
- LEAST_SCHEMA_REGION_GROUP_NUM,
- Math.max(
- (int)
- // Use Math.ceil here to ensure that the
maxSchemaRegionGroupNum
- // will be increased as long as the number of cluster
DataNodes is increased
- Math.ceil(
- // The maxSchemaRegionGroupNum of the current
StorageGroup
- // is expected to be:
- // (SCHEMA_REGION_PER_DATA_NODE *
registerDataNodeNum) /
- // (createdStorageGroupNum *
schemaReplicationFactor)
- SCHEMA_REGION_PER_DATA_NODE
- * dataNodeNum
- / storageGroupNum
- *
storageGroupSchema.getSchemaReplicationFactor()),
- allocatedSchemaRegionGroupCount));
+ calcMaxRegionGroupNum(
+ CONF.getLeastSchemaRegionGroupNum(),
+ SCHEMA_REGION_PER_DATA_NODE,
+ dataNodeNum,
+ storageGroupNum,
+ storageGroupSchema.getSchemaReplicationFactor(),
+ allocatedSchemaRegionGroupCount);
LOGGER.info(
"[AdjustRegionGroupNum] The maximum number of SchemaRegionGroups
for Database: {} is adjusted to: {}",
storageGroupSchema.getName(),
@@ -381,24 +373,13 @@ public class ClusterSchemaManager {
getPartitionManager()
.getRegionGroupCount(storageGroupSchema.getName(),
TConsensusGroupType.DataRegion);
int maxDataRegionGroupNum =
- Math.max(
- // The least number of DataRegionGroup of each StorageGroup is
specified
- // by parameter least_data_region_group_num.
- LEAST_DATA_REGION_GROUP_NUM,
- Math.max(
- (int)
- // Use Math.ceil here to ensure that the
maxDataRegionGroupNum
- // will be increased as long as the number of cluster
DataNodes is increased
- Math.ceil(
- // The maxDataRegionGroupNum of the current
StorageGroup
- // is expected to be:
- // (DATA_REGION_PER_PROCESSOR * totalCpuCoreNum) /
- // (createdStorageGroupNum * dataReplicationFactor)
- DATA_REGION_PER_PROCESSOR
- * totalCpuCoreNum
- / storageGroupNum
- *
storageGroupSchema.getDataReplicationFactor()),
- allocatedDataRegionGroupCount));
+ calcMaxRegionGroupNum(
+ CONF.getLeastDataRegionGroupNum(),
+ DATA_REGION_PER_PROCESSOR,
+ totalCpuCoreNum,
+ storageGroupNum,
+ storageGroupSchema.getDataReplicationFactor(),
+ allocatedDataRegionGroupCount);
LOGGER.info(
"[AdjustRegionGroupNum] The maximum number of DataRegionGroups for
Database: {} is adjusted to: {}",
storageGroupSchema.getName(),
@@ -414,6 +395,28 @@ public class ClusterSchemaManager {
getConsensusManager().write(adjustMaxRegionGroupNumPlan);
}
+ public static int calcMaxRegionGroupNum(
+ int leastRegionGroupNum,
+ double resourceWeight,
+ int resource,
+ int storageGroupNum,
+ int replicationFactor,
+ int allocatedRegionGroupCount) {
+ return Math.max(
+ // The maxRegionGroupNum should be great or equal to the
leastRegionGroupNum
+ leastRegionGroupNum,
+ Math.max(
+ (int)
+ // Use Math.ceil here to ensure that the maxRegionGroupNum
+ // will be increased as long as the number of cluster
DataNodes is increased
+ Math.ceil(
+ // The maxRegionGroupNum of the current StorageGroup is
expected to be:
+ // (resourceWeight * resource) / (createdStorageGroupNum *
replicationFactor)
+ resourceWeight * resource / (double) (storageGroupNum *
replicationFactor)),
+ // The maxRegionGroupNum should be great or equal to the
allocatedRegionGroupCount
+ allocatedRegionGroupCount));
+ }
+
// ======================================================
// Leader scheduling interfaces
// ======================================================
diff --git
a/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java
b/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java
new file mode 100644
index 0000000000..2af715eb1b
--- /dev/null
+++
b/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.manager;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ClusterSchemaManagerTest {
+
+ @Test
+ public void testCalcMaxRegionGroupNum() {
+
+ // The maxRegionGroupNum should be great or equal to the
leastRegionGroupNum
+ Assert.assertEquals(100, ClusterSchemaManager.calcMaxRegionGroupNum(100,
1.0, 3, 1, 3, 0));
+
+ // The maxRegionGroupNum should be great or equal to the
allocatedRegionGroupCount
+ Assert.assertEquals(100, ClusterSchemaManager.calcMaxRegionGroupNum(3,
1.0, 6, 2, 3, 100));
+
+ // (resourceWeight * resource) / (createdStorageGroupNum *
replicationFactor)
+ Assert.assertEquals(20, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0,
120, 2, 3, 5));
+ }
+}
