This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch cherrypick_partition in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ca4f974e4197ad641bac0639558e2b819edcb2c1 Author: YongzaoDan <[email protected]> AuthorDate: Tue May 30 00:56:45 2023 +0800 [IOTDB-5934] Optimize cluster partition policy (#9971) * Update PartitionManager.java * finish * Fix IT bug --- .../apache/iotdb/confignode/conf/ConfigNodeConfig.java | 4 ++-- .../confignode/manager/partition/PartitionManager.java | 16 +++++++++++++++- .../iotdb/confignode/it/utils/ConfigNodeTestUtils.java | 2 +- .../src/assembly/resources/conf/iotdb-common.properties | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 22f2e86a196..a6a878da47d 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -68,8 +68,8 @@ public class ConfigNodeConfig { /** Default number of DataRegion replicas */ private int dataReplicationFactor = 1; - /** Number of SeriesPartitionSlots per StorageGroup */ - private int seriesSlotNum = 10000; + /** Number of SeriesPartitionSlots per Database */ + private int seriesSlotNum = 1000; /** SeriesPartitionSlot executor class */ private String seriesPartitionExecutorClass = diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index c90e510bbdc..71407eeff33 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -208,6 +208,13 @@ public class PartitionManager { // by the number of SeriesPartitionSlots, // the number of serialized CreateSchemaPartitionReqs is acceptable. synchronized (this) { + // Here we should check again if the SchemaPartition + // has been created by other threads to improve concurrent performance + resp = (SchemaPartitionResp) getSchemaPartition(req); + if (resp.isAllPartitionsExist()) { + return resp; + } + // Filter unassigned SchemaPartitionSlots Map<String, List<TSeriesPartitionSlot>> unassignedSchemaPartitionSlotsMap = partitionInfo.filterUnassignedSchemaPartitionSlots(req.getPartitionSlotsMap()); @@ -261,7 +268,7 @@ public class PartitionManager { resp = (SchemaPartitionResp) getSchemaPartition(req); if (!resp.isAllPartitionsExist()) { LOGGER.error( - "Lacked some SchemaPartition allocation result in the response of getOrCreateDataPartition method"); + "Lacked some SchemaPartition allocation result in the response of getOrCreateSchemaPartition method"); resp.setStatus( new TSStatus(TSStatusCode.LACK_PARTITION_ALLOCATION.getStatusCode()) .setMessage("Lacked some SchemaPartition allocation result in the response")); @@ -307,6 +314,13 @@ public class PartitionManager { // by the number of SeriesPartitionSlots, // the number of serialized CreateDataPartitionReqs is acceptable. synchronized (this) { + // Here we should check again if the DataPartition + // has been created by other threads to improve concurrent performance + resp = (DataPartitionResp) getDataPartition(req); + if (resp.isAllPartitionsExist()) { + return resp; + } + // Filter unassigned DataPartitionSlots Map<String, Map<TSeriesPartitionSlot, TTimeSlotList>> unassignedDataPartitionSlotsMap = partitionInfo.filterUnassignedDataPartitionSlots(req.getPartitionSlotsMap()); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java index b3123838fcf..e1588956e2f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java @@ -201,7 +201,7 @@ public class ConfigNodeTestUtils { "org.apache.iotdb.consensus.simple.SimpleConsensus"); clusterParameters.setSchemaRegionConsensusProtocolClass( "org.apache.iotdb.consensus.simple.SimpleConsensus"); - clusterParameters.setSeriesPartitionSlotNum(10000); + clusterParameters.setSeriesPartitionSlotNum(1000); clusterParameters.setSeriesPartitionExecutorClass( "org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor"); clusterParameters.setDefaultTTL(Long.MAX_VALUE); diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties index 75f1c972fb5..e9a62e60ae5 100644 --- a/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -71,7 +71,7 @@ cluster_name=defaultCluster # And these parameters should be consistent within the ConfigNodeGroup. # Number of SeriesPartitionSlots per Database # Datatype: Integer -# series_slot_num=10000 +# series_slot_num=1000 # SeriesPartitionSlot executor class # These hashing algorithms are currently supported:
