This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 33c460d7365 [fix](fe) Check automatic partition limit before creation
(#69128)
33c460d7365 is described below
commit 33c460d7365f7bba0111b01d2ee3adc82097669f
Author: Jerry Hu <[email protected]>
AuthorDate: Tue Oct 20 18:54:18 2026 +0800
[fix](fe) Check automatic partition limit before creation (#69128)
### What problem does this PR solve?
Issue Number: None
Automatic partition creation currently checks `max_auto_partition_num`
only after creating and logging every requested partition. If a load
brings more distinct partition values than the remaining capacity, the
RPC fails but the new partitions remain; repeated bad input can keep
growing table metadata beyond the configured limit.
This change counts only missing partition names before starting the
batch and rejects an over-limit request without creating any partitions.
It also checks the count under the table write lock immediately before
publication, so concurrent requests cannot publish an automatic
partition beyond the then-current limit. Existing-name requests at the
limit remain valid, and explicit partition DDL keeps its prior behavior.
A concurrent request can exhaust the remaining capacity partway through
another multi-partition batch. In that case, partitions already
published within the limit are not rolled back; the change enforces the
publication cap, not atomicity of the whole batch. It does not remove
existing oversized tables or change the image serialization format.
### Release note
Automatic partition requests exceeding `max_auto_partition_num` are
rejected before creating partitions in the normal batch path.
### Check List (For Author)
- Test:
- FE Unit Test:
`FrontendServiceImplTest#testCreatePartitionRangeLimitDoesNotLeavePartitions+testCreatePartitionListLimitCountsOnlyNewPartitions+testCreatePartitionLimitIsRecheckedAtPublication`
(3 passed)
- FE build: `DORIS_THIRDPARTY=<compatible installed snapshot> ./build.sh
--fe` (passed; fe-core Checkstyle 0 violations)
- SQL regression and real multi-threaded load test: not run; the
targeted FE RPC tests cover the limit and deterministic stale-precheck
boundary.
- Behavior changed: Yes. Over-limit automatic partition requests no
longer leave new partitions in the ordinary batch path.
- Does this need documentation: No.
---
.../main/java/org/apache/doris/catalog/Env.java | 12 ++-
.../apache/doris/datasource/InternalCatalog.java | 22 +++++
.../apache/doris/service/FrontendServiceImpl.java | 32 ++++++-
.../doris/service/FrontendServiceImplTest.java | 102 +++++++++++++++++++++
4 files changed, 162 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 2c041999427..7338b6727a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3851,8 +3851,18 @@ public class Env {
boolean writeEditLog,
List<Pair<PartitionPersistInfo,
Partition>> batchPartitions)
throws DdlException {
+ addPartition(db, tableName, addPartitionOp, isCreateTable,
generatedPartitionId,
+ writeEditLog, batchPartitions, false);
+ }
+
+ public void addPartition(Database db, String tableName, AddPartitionOp
addPartitionOp,
+ boolean isCreateTable, long
generatedPartitionId,
+ boolean writeEditLog,
+ List<Pair<PartitionPersistInfo,
Partition>> batchPartitions,
+ boolean enforceAutoPartitionLimit)
+ throws DdlException {
getInternalCatalog().addPartition(db, tableName, addPartitionOp,
- isCreateTable, generatedPartitionId, writeEditLog,
batchPartitions);
+ isCreateTable, generatedPartitionId, writeEditLog,
batchPartitions, enforceAutoPartitionLimit);
}
public void addMultiPartitions(Database db, String tableName,
AlterMultiPartitionOp multiPartitionOp)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index da9dbeeed0b..49dc07252e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1527,6 +1527,16 @@ public class InternalCatalog implements
CatalogIf<Database> {
boolean writeEditLog,
List<Pair<PartitionPersistInfo,
Partition>> batchPartitions)
throws DdlException {
+ addPartition(db, tableName, addPartitionOp, isCreateTable,
generatedPartitionId,
+ writeEditLog, batchPartitions, false);
+ }
+
+ public void addPartition(Database db, String tableName, AddPartitionOp
addPartitionOp,
+ boolean isCreateTable, long
generatedPartitionId,
+ boolean writeEditLog,
+ List<Pair<PartitionPersistInfo,
Partition>> batchPartitions,
+ boolean enforceAutoPartitionLimit)
+ throws DdlException {
// in cloud mode, isCreateTable == true, create dynamic partition use,
so partitionId must have been generated.
// isCreateTable == false, other case, partitionId generate in below,
must be set 0
if (!FeConstants.runningUnitTest && Config.isCloudMode()
@@ -1809,6 +1819,18 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
}
+ // Check at publication under the table write lock. A request
may have passed the
+ // batch precheck before another request published its own
auto partitions.
+ if (enforceAutoPartitionLimit) {
+ int partitionNum = olapTable.getPartitionNum();
+ int autoPartitionLimit = Config.max_auto_partition_num;
+ if (partitionNum >= autoPartitionLimit) {
+ throw new DdlException(String.format(
+ "partition numbers %d exceeded limit of
variable max_auto_partition_num %d",
+ (long) partitionNum + 1, autoPartitionLimit));
+ }
+ }
+
// check if meta changed
// rollup index may be added or dropped during add partition
operation.
// schema may be changed during add partition operation.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 4825fd767ce..9ea0c09e5a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -4801,10 +4801,34 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
return result;
}
+ olapTable.readLock();
+ try {
+ int partitionNum = olapTable.getPartitionNum();
+ int newPartitionNum = 0;
+ for (String partitionName : addPartitionClauseMap.keySet()) {
+ if (!olapTable.checkPartitionNameExist(partitionName)) {
+ newPartitionNum++;
+ }
+ }
+ long expectedPartitionNum = (long) partitionNum + newPartitionNum;
+ int autoPartitionLimit = Config.max_auto_partition_num;
+ if (expectedPartitionNum > autoPartitionLimit) {
+ String errorMessage = String.format(
+ "partition numbers %d exceeded limit of variable
max_auto_partition_num %d",
+ expectedPartitionNum, autoPartitionLimit);
+ LOG.warn(errorMessage);
+ errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage));
+ result.setStatus(errorStatus);
+ LOG.warn("send create partition error status: {}", result);
+ return result;
+ }
+ } finally {
+ olapTable.readUnlock();
+ }
+
for (AddPartitionOp addPartitionOp : addPartitionClauseMap.values()) {
try {
- // here maybe check and limit created partitions num
- Env.getCurrentEnv().addPartition(db, olapTable.getName(),
addPartitionOp, false, 0, true, null);
+ Env.getCurrentEnv().addPartition(db, olapTable.getName(),
addPartitionOp, false, 0, true, null, true);
} catch (DdlException e) {
LOG.warn(e);
errorStatus.setErrorMsgs(
@@ -4829,9 +4853,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
olapTable.readLock();
try {
- // check partition's number limit. because partitions in
addPartitionClauseMap may be duplicated with
- // existing partitions, which would lead to false positive. so we
should check the partition number AFTER
- // adding new partitions using its ACTUAL NUMBER, rather than the
sum of existing and requested partitions.
+ // Retain the actual-count check for concurrent changes and
configuration updates during creation.
int partitionNum = olapTable.getPartitionNum();
int autoPartitionLimit = Config.max_auto_partition_num;
if (partitionNum > autoPartitionLimit) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
index 543ab7ebae7..21632f3594c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
@@ -558,6 +558,108 @@ public class FrontendServiceImplTest extends
TestWithFeService {
Assertions.assertNotNull(p20230807);
}
+ @Test
+ public void testCreatePartitionRangeLimitDoesNotLeavePartitions() throws
Exception {
+ createTable("CREATE TABLE test.partition_range_limit (event_day
DATETIME NOT NULL, id INT) "
+ + "DUPLICATE KEY(event_day, id) "
+ + "AUTO PARTITION BY RANGE (date_trunc(event_day, 'day')) () "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num' = '1')");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("partition_range_limit");
+ int oldLimit = Config.max_auto_partition_num;
+ try {
+ Config.max_auto_partition_num = 1;
+ TCreatePartitionResult result = new
FrontendServiceImpl(exeEnv).createPartition(
+ createPartitionRequest(db, table, "2023-08-07 00:00:00",
"2023-08-08 00:00:00"));
+ Assertions.assertEquals(TStatusCode.RUNTIME_ERROR,
result.getStatus().getStatusCode());
+
Assertions.assertTrue(result.getStatus().getErrorMsgs().get(0).contains("max_auto_partition_num"));
+ Assertions.assertEquals(0, table.getPartitionNum());
+ } finally {
+ Config.max_auto_partition_num = oldLimit;
+ }
+ }
+
+ @Test
+ public void testCreatePartitionListLimitCountsOnlyNewPartitions() throws
Exception {
+ createTable("CREATE TABLE test.partition_list_limit (city_code
VARCHAR(100) NOT NULL, id INT) "
+ + "DUPLICATE KEY(city_code, id) "
+ + "AUTO PARTITION BY LIST (city_code) () "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num' = '1')");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("partition_list_limit");
+ int oldLimit = Config.max_auto_partition_num;
+ try {
+ Config.max_auto_partition_num = 1;
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TCreatePartitionResult first =
impl.createPartition(createPartitionRequest(db, table, "BEIJING"));
+ Assertions.assertEquals(TStatusCode.OK,
first.getStatus().getStatusCode());
+ Assertions.assertEquals(1, table.getPartitionNum());
+
+ TCreatePartitionResult existing =
impl.createPartition(createPartitionRequest(db, table, "BEIJING"));
+ Assertions.assertEquals(TStatusCode.OK,
existing.getStatus().getStatusCode());
+ Assertions.assertEquals(1, table.getPartitionNum());
+
+ TCreatePartitionResult overflow = impl.createPartition(
+ createPartitionRequest(db, table, "BEIJING", "SHANGHAI"));
+ Assertions.assertEquals(TStatusCode.RUNTIME_ERROR,
overflow.getStatus().getStatusCode());
+
Assertions.assertTrue(overflow.getStatus().getErrorMsgs().get(0).contains("max_auto_partition_num"));
+ Assertions.assertEquals(1, table.getPartitionNum());
+ } finally {
+ Config.max_auto_partition_num = oldLimit;
+ }
+ }
+
+ @Test
+ public void testCreatePartitionLimitIsRecheckedAtPublication() throws
Exception {
+ createTable("CREATE TABLE test.partition_list_concurrent_limit
(city_code VARCHAR(100) NOT NULL, id INT) "
+ + "DUPLICATE KEY(city_code, id) "
+ + "AUTO PARTITION BY LIST (city_code) () "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num' = '1')");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("partition_list_concurrent_limit");
+ int oldLimit = Config.max_auto_partition_num;
+ try {
+ Config.max_auto_partition_num = 1;
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TCreatePartitionResult first =
impl.createPartition(createPartitionRequest(db, table, "BEIJING"));
+ Assertions.assertEquals(TStatusCode.OK,
first.getStatus().getStatusCode());
+
+ // Model a concurrent publication after the request's read-lock
precheck.
+ OlapTable spyTable = Mockito.spy(table);
+ AtomicBoolean stalePrecheck = new AtomicBoolean(true);
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ Mockito.doAnswer(invocation ->
stalePrecheck.compareAndSet(true, false)
+ ? 0 :
invocation.callRealMethod()).when(spyTable).getPartitionNum();
+ TCreatePartitionResult result =
impl.createPartition(createPartitionRequest(db, spyTable, "SHANGHAI"));
+ Assertions.assertEquals(TStatusCode.RUNTIME_ERROR,
result.getStatus().getStatusCode());
+
Assertions.assertTrue(result.getStatus().getErrorMsgs().get(0).contains("max_auto_partition_num"));
+ Assertions.assertFalse(stalePrecheck.get());
+ Assertions.assertEquals(1, spyTable.getPartitionNum());
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ } finally {
+ Config.max_auto_partition_num = oldLimit;
+ }
+ }
+
+ private static TCreatePartitionRequest createPartitionRequest(Database db,
OlapTable table, String... values) {
+ List<List<TNullableStringLiteral>> partitionValues = new ArrayList<>();
+ for (String value : values) {
+ TNullableStringLiteral literal = new TNullableStringLiteral();
+ literal.setValue(value);
+ partitionValues.add(Collections.singletonList(literal));
+ }
+ TCreatePartitionRequest request = new TCreatePartitionRequest();
+ request.setDbId(db.getId());
+ request.setTableId(table.getId());
+ request.setPartitionValues(partitionValues);
+ return request;
+ }
+
@Test
public void testCreatePartitionWithRowBinlog() throws Exception {
String createOlapTblStmt = "CREATE TABLE
test.partition_range_with_row_binlog(\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
