This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch rc/1.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.0.1 by this push:
new 63b16f215a [To rc/1.0.1][IOTDB-5452] Add
setLeastDataRegionGroupNumSetByUser judgement (#8959)
63b16f215a is described below
commit 63b16f215a78925fa779dce387e9c2eb3d3b950d
Author: Beyyes <[email protected]>
AuthorDate: Thu Feb 2 13:53:32 2023 +0800
[To rc/1.0.1][IOTDB-5452] Add setLeastDataRegionGroupNumSetByUser judgement
(#8959)
---
.github/workflows/main-win.yml | 47 ++--------------------
.../iotdb/confignode/conf/ConfigNodeConfig.java | 11 +++++
.../confignode/conf/ConfigNodeDescriptor.java | 9 +++--
.../confignode/manager/ClusterSchemaManager.java | 26 ++++++------
4 files changed, 33 insertions(+), 60 deletions(-)
diff --git a/.github/workflows/main-win.yml b/.github/workflows/main-win.yml
index 7fd9f55ecc..33c6ac0f87 100644
--- a/.github/workflows/main-win.yml
+++ b/.github/workflows/main-win.yml
@@ -45,9 +45,7 @@ jobs:
# the server module:
# -pl server -am -DskipTests=true -Diotdb.test.only=true
# but we just add labels here to make the action name more graceful
- it_task: [ 'others',
- 'server'
- ]
+ it_task: [ 'others', 'server']
runs-on: windows-latest
steps:
@@ -65,51 +63,12 @@ jobs:
restore-keys: ${{ runner.os }}-m2-
- name: Check Apache Rat
run: mvn -B apache-rat:check
- - name: Download Hadoop Tools if Needed
- # Using Powershell ...
- # github action put codes to d:\a
- run: mkdir D:\a\hadoop\bin ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/winutils.exe
-OutFile D:\a\hadoop\bin\winutils.exe ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hadoop.dll
-OutFile C:\windows\system32\hadoop.dll ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hadoop
-OutFile D:\a\hadoop\bin\hadoop ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hadoop.cmd
-OutFile D:\a\hadoop\bin\hadoop.cmd ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hadoop.exp
-OutFile D:\a\hadoop\bin\hadoop.exp ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hadoop.lib
-OutFile D:\a\hadoop\bin\hadoop.lib ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hadoop.pdb
-OutFile D:\a\hadoop\bin\hadoop.pdb ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs.dll
-OutFile C:\windows\system32\hdfs.dll ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs -OutFile
D:\a\hadoop\bin\hdfs ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs.cmd
-OutFile D:\a\hadoop\bin\hdfs.cmd ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs.exp
-OutFile D:\a\hadoop\bin\hdfs.exp ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs.lib
-OutFile D:\a\hadoop\bin\hdfs.lib ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs.pdb
-OutFile D:\a\hadoop\bin\hdfs.pdb ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/hdfs_static.lib
-OutFile D:\a\hadoop\bin\hdfs_static.lib ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/libwinutils.lib
-OutFile D:\a\hadoop\bin\libwinutils.lib ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/mapred
-OutFile D:\a\hadoop\bin\mapred ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/mapred.cmd
-OutFile D:\a\hadoop\bin\mapred.cmd ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/rcc -OutFile
D:\a\hadoop\bin\rcc ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/winutils.pdb
-OutFile D:\a\hadoop\bin\winutils.pdb ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/yarn -OutFile
D:\a\hadoop\bin\yarn ; `
- Invoke-WebRequest
https://github.com/cdarlint/winutils/raw/master/hadoop-2.7.3/bin/yarn.cmd
-OutFile D:\a\hadoop\bin\yarn.cmd ; `
- [Environment]::SetEnvironmentVariable("Path", $env:Path +
";D:\a\hadoop\bin", "User") ; `
- [Environment]::SetEnvironmentVariable("HADOOP_HOME", "D:\a\hadoop",
"User")
- - name: Add Hadoop Path (Win)
- shell: bash
- run: echo 'export PATH=/d/a/hadoop/bin:$PATH' >> ~/.bash_profile &&
echo 'export HADOOP_HOME=/d/a/hadoop' >> ~/.bash_profile && source
~/.bash_profile
- - name: Check Hadoop Path (Win)
- shell: bash
- run: source ~/.bash_profile && echo $PATH
- - name: Cache Hadoop packages (Win)
- uses: actions/cache@v3
- with:
- path: D:\a\hadoop
- key: ${{ runner.os }}-hadoop-files${{ hashFiles('**/*.exe') }}
- restore-keys: ${{ runner.os }}-hadoop-files
- name: Test Server Module with Maven
shell: bash
if: ${{ matrix.it_task == 'server'}}
- run: source ~/.bash_profile && mvn -B clean integration-test
-Dtest.port.closed=true -pl server -am -DskipTests=true -Diotdb.test.only=true
+ run: mvn clean integration-test -Dtest.port.closed=true -pl server -am
-DskipTests=true -Diotdb.test.only=true
- name: Test Other Modules with Maven
shell: bash
if: ${{ matrix.it_task == 'others'}}
- run: source ~/.bash_profile && mvn -B clean test
-Dtest.port.closed=true -Diotdb.test.skip=true -P get-jar-with-dependencies
+ run: mvn clean test -Dtest.port.closed=true -Diotdb.test.skip=true -P
get-jar-with-dependencies
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 4be8758fde..63bd4f02fa 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -100,6 +100,9 @@ public class ConfigNodeConfig {
/** The least number of DataRegionGroup for each Database. */
private volatile int leastDataRegionGroupNum = 5;
+ /** Indicate whether the leastDataRegionGroupNum is set by user, if true,
lock it. */
+ private volatile boolean leastDataRegionGroupNumSetByUser = false;
+
/** RegionGroup allocate policy. */
private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
RegionBalancer.RegionGroupAllocatePolicy.GREEDY;
@@ -507,6 +510,14 @@ public class ConfigNodeConfig {
this.leastDataRegionGroupNum = leastDataRegionGroupNum;
}
+ public boolean isLeastDataRegionGroupNumSetByUser() {
+ return leastDataRegionGroupNumSetByUser;
+ }
+
+ public void setLeastDataRegionGroupNumSetByUser(boolean
leastDataRegionGroupNumSetByUser) {
+ this.leastDataRegionGroupNumSetByUser = leastDataRegionGroupNumSetByUser;
+ }
+
public RegionBalancer.RegionGroupAllocatePolicy
getRegionGroupAllocatePolicy() {
return regionGroupAllocatePolicy;
}
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index 63d30fc6cd..ca4d7e6b25 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -254,10 +254,11 @@ public class ConfigNodeDescriptor {
"data_region_per_processor",
String.valueOf(conf.getDataRegionPerProcessor()))
.trim()));
- conf.setLeastDataRegionGroupNum(
- Integer.parseInt(
- properties.getProperty(
- "least_data_region_group_num",
String.valueOf(conf.getLeastDataRegionGroupNum()))));
+ if (properties.getProperty("least_data_region_group_num") != null) {
+ conf.setLeastDataRegionGroupNum(
+
Integer.parseInt(properties.getProperty("least_data_region_group_num")));
+ conf.setLeastDataRegionGroupNumSetByUser(true);
+ }
try {
conf.setRegionAllocateStrategy(
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
index 91746ace72..6e8250b87d 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
@@ -326,18 +326,20 @@ public class ClusterSchemaManager {
// Adjust least_data_region_group_num
// TODO: The least_data_region_group_num should be maintained separately
by different
// TODO: StorageGroup
- int leastDataRegionGroupNum =
- (int)
- Math.ceil(
- (double) totalCpuCoreNum
- / (double) (storageGroupNum *
CONF.getDataReplicationFactor()));
- if (leastDataRegionGroupNum < CONF.getLeastDataRegionGroupNum()) {
- // The leastDataRegionGroupNum should be the maximum integer that
satisfy:
- // 1 <= leastDataRegionGroupNum <= 5(default)
- CONF.setLeastDataRegionGroupNum(leastDataRegionGroupNum);
- LOGGER.info(
- "[AdjustRegionGroupNum] The least number of DataRegionGroups per
Database is adjusted to: {}",
- leastDataRegionGroupNum);
+ if (!CONF.isLeastDataRegionGroupNumSetByUser()) {
+ int leastDataRegionGroupNum =
+ (int)
+ Math.ceil(
+ (double) totalCpuCoreNum
+ / (double) (storageGroupNum *
CONF.getDataReplicationFactor()));
+ if (leastDataRegionGroupNum < CONF.getLeastDataRegionGroupNum()) {
+ // The leastDataRegionGroupNum should be the maximum integer that
satisfy:
+ // 1 <= leastDataRegionGroupNum <= 5(default)
+ CONF.setLeastDataRegionGroupNum(leastDataRegionGroupNum);
+ LOGGER.info(
+ "[AdjustRegionGroupNum] The least number of DataRegionGroups per
Database is adjusted to: {}",
+ leastDataRegionGroupNum);
+ }
}
AdjustMaxRegionGroupNumPlan adjustMaxRegionGroupNumPlan = new
AdjustMaxRegionGroupNumPlan();
