This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 44da2ee1058 [opt](partition) Add global dynamic partition drop
protection (#68844)
44da2ee1058 is described below
commit 44da2ee105878448b40705e1029861d4b9f71a8c
Author: Gavin Chou <[email protected]>
AuthorDate: Tue Oct 13 14:16:31 2026 +0800
[opt](partition) Add global dynamic partition drop protection (#68844)
Problem Summary: Some operational workflows need a cluster-wide guard
that temporarily prevents background dynamic partition cleanup while
allowing partition creation to continue.
### Release note
Add the global `enable_dynamic_partition_drop_protection` variable to
pause automatic partition drops without disabling dynamic partition
creation.
---
.../doris/clone/DynamicPartitionScheduler.java | 30 +++++--
.../java/org/apache/doris/qe/SessionVariable.java | 14 ++++
.../main/java/org/apache/doris/qe/VariableMgr.java | 30 +++++++
.../doris/catalog/DynamicPartitionTableTest.java | 97 ++++++++++++++++++++++
.../java/org/apache/doris/qe/VariableMgrTest.java | 26 ++++++
5 files changed, 191 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index d552d0cb88e..c467bb6765f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -59,6 +59,8 @@ import
org.apache.doris.nereids.trees.plans.commands.info.AddPartitionOp;
import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionOp;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.persist.PartitionPersistInfo;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.VariableMgr;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.annotations.VisibleForTesting;
@@ -787,13 +789,17 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
executeFirstTime);
}
clearDropPartitionFailedMsg(olapTable.getId());
- if (olapTable.getPartitionRetentionCount() > 0) {
- // Handle AUTO PARTITION cleanup based on
partition.retention_count
- dropPartitionOps = getDropPartitionOpForAutoPartition(db,
olapTable);
+ if (VariableMgr.isDynamicPartitionDropProtectionEnabled()) {
+ logDynamicPartitionDropProtection(db, olapTable);
} else {
- // Handle dynamic partition cleanup
- dropPartitionOps = getDropPartitionOpForDynamic(db,
olapTable, partitionColumn,
- partitionFormat);
+ if (olapTable.getPartitionRetentionCount() > 0) {
+ // Handle AUTO PARTITION cleanup based on
partition.retention_count
+ dropPartitionOps =
getDropPartitionOpForAutoPartition(db, olapTable);
+ } else {
+ // Handle dynamic partition cleanup
+ dropPartitionOps = getDropPartitionOpForDynamic(db,
olapTable, partitionColumn,
+ partitionFormat);
+ }
}
tableName = olapTable.getName();
} catch (Exception e) {
@@ -811,6 +817,10 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
continue;
}
try {
+ if (VariableMgr.isDynamicPartitionDropProtectionEnabled())
{
+ logDynamicPartitionDropProtection(db, olapTable);
+ break;
+ }
Env.getCurrentEnv().dropPartition(db, olapTable,
dropPartitionOp);
} catch (Exception e) {
recordDropPartitionFailedMsg(db.getFullName(), tableName,
e.getMessage(), olapTable.getId());
@@ -1013,6 +1023,14 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
createOrUpdateRuntimeInfo(tableId, CREATE_PARTITION_MSG,
DEFAULT_RUNTIME_VALUE);
}
+ private void logDynamicPartitionDropProtection(Database db, OlapTable
olapTable) {
+ LOG.warn("Skip automatic partition drop for db [{}-{}], table [{}-{}]
because global session variable "
+ + "'{}' is enabled. Existing partitions are retained;
set it to false to resume automatic "
+ + "partition cleanup",
+ db.getId(), db.getName(), olapTable.getId(),
olapTable.getName(),
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION);
+ }
+
private void recordDropPartitionFailedMsg(String dbName, String tableName,
String msg, long tableId) {
LOG.warn("dynamic drop partition failed: {}, db: {}, table: {}", msg,
dbName, tableName);
createOrUpdateRuntimeInfo(tableId, DYNAMIC_PARTITION_STATE,
State.ERROR.toString());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 1cdfc01674a..3f04a2b4ac2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -204,6 +204,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_REWRITE_ELEMENT_AT_TO_SLOT =
"enable_rewrite_element_at_to_slot";
public static final String ENABLE_ODBC_TRANSCATION =
"enable_odbc_transcation";
public static final String ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS =
"enable_binary_search_filtering_partitions";
+ public static final String ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION =
+ "enable_dynamic_partition_drop_protection";
public static final String CACHE_SORTED_PARTITION_INTERVAL_SECOND =
"cache_sorted_partition_interval_second";
public static final String SKIP_PRUNE_PREDICATE = "skip_prune_predicate";
public static final String ENABLE_SQL_CACHE = "enable_sql_cache";
@@ -1531,6 +1533,18 @@ public class SessionVariable implements Serializable,
Writable {
)
public boolean enableBinarySearchFilteringPartitions = true;
+ @VarAttrDef.VarAttr(
+ name = ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
+ flag = VarAttrDef.GLOBAL,
+ description = "Whether to prevent background dynamic partition
tasks from automatically dropping "
+ + "partitions. OFF by default."
+ )
+ private boolean enableDynamicPartitionDropProtection = false;
+
+ public boolean isEnableDynamicPartitionDropProtection() {
+ return enableDynamicPartitionDropProtection;
+ }
+
@VarAttrDef.VarAttr(
name = CACHE_SORTED_PARTITION_INTERVAL_SECOND,
fuzzy = false,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
index 5d2e505684a..ec4bc17148e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -178,6 +178,36 @@ public class VariableMgr {
return defaultSessionVariable;
}
+ public static boolean isDynamicPartitionDropProtectionEnabled() {
+ rlock.lock();
+ try {
+ return
defaultSessionVariable.isEnableDynamicPartitionDropProtection();
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ /**
+ * Sets a persistent global-only session variable while constructing an
image.
+ *
+ * <p>The caller is responsible for persisting the resulting image. This
method intentionally does not write an
+ * edit log because image construction may happen without a running edit
log.</p>
+ */
+ public static void setGlobalSessionVariableForImage(String name, String
value) throws DdlException {
+ wlock.lock();
+ try {
+ VarContext ctx = getVarContext(name);
+ Preconditions.checkArgument(ctx != null, "Unknown session
variable: %s", name);
+ Preconditions.checkArgument(ctx.getObj() == defaultSessionVariable,
+ "Not a session variable: %s", name);
+ Preconditions.checkArgument((ctx.getFlag() & VarAttrDef.GLOBAL) !=
0,
+ "Session variable is not global-only: %s", name);
+ setValue(defaultSessionVariable, new
SessionVariableField(ctx.getField()), value);
+ } finally {
+ wlock.unlock();
+ }
+ }
+
// Set value to a variable
private static boolean setValue(Object obj, SessionVariableField
sessionVariableField, String value)
throws DdlException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index eee11b90fbf..28240e22bb2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -30,18 +30,22 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.mysql.authenticate.TestLogAppender;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.qe.VariableMgr;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
+import org.apache.logging.log4j.Level;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@@ -108,6 +112,99 @@ public class DynamicPartitionTableTest extends
TestWithFeService {
}
}
+ private void setDynamicPartitionDropProtection(boolean enabled) throws
DdlException {
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
Boolean.toString(enabled));
+ }
+
+ @Test
+ public void testDynamicPartitionDropProtection() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_drop_protection` (\n"
+ + " `k1` date NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(`k1`)\n"
+ + "PARTITION BY RANGE (`k1`) ()\n"
+ + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"dynamic_partition.enable\" = \"false\",\n"
+ + "\"dynamic_partition.start\" = \"-1\",\n"
+ + "\"dynamic_partition.end\" = \"1\",\n"
+ + "\"dynamic_partition.time_unit\" = \"day\",\n"
+ + "\"dynamic_partition.prefix\" = \"p\",\n"
+ + "\"dynamic_partition.buckets\" = \"1\"\n"
+ + ");";
+ createTableStmt(createOlapTblStmt);
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("dynamic_partition_drop_protection");
+ alterTable("ALTER TABLE test.dynamic_partition_drop_protection ADD
PARTITION p_old "
+ + "VALUES [('2000-01-01'), ('2000-01-02'))");
+
+ try {
+ setDynamicPartitionDropProtection(true);
+ alterTable("ALTER TABLE test.dynamic_partition_drop_protection SET
"
+ + "('dynamic_partition.enable' = 'true')");
+ try (TestLogAppender appender =
TestLogAppender.attach(DynamicPartitionScheduler.class, Level.WARN)) {
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+ Assertions.assertTrue(appender.contains(Level.WARN,
+ "table [" + table.getId() + "-" + table.getName()
+ + "] because global session variable '"
+ +
SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION + "' is enabled"));
+ }
+
+ Assertions.assertTrue(table.getPartitionNames().contains("p_old"));
+ Assertions.assertTrue(table.getPartitionNames().size() > 1,
+ "Dynamic partition creation should continue while drop
protection is enabled");
+
+ setDynamicPartitionDropProtection(false);
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+
Assertions.assertFalse(table.getPartitionNames().contains("p_old"));
+ } finally {
+ setDynamicPartitionDropProtection(false);
+ }
+ }
+
+ @Test
+ public void testAutoPartitionRetentionDropProtection() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`auto_partition_drop_protection` (\n"
+ + " `k1` datetime NOT NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(`k1`)\n"
+ + "AUTO PARTITION BY RANGE (date_trunc(k1, 'day')) ()\n"
+ + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"partition.retention_count\" = \"1\"\n"
+ + ");";
+ createTableStmt(createOlapTblStmt);
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("auto_partition_drop_protection");
+ alterTable("ALTER TABLE test.auto_partition_drop_protection ADD
PARTITION p_old "
+ + "VALUES [('2000-01-01'), ('2000-01-02'))");
+ alterTable("ALTER TABLE test.auto_partition_drop_protection ADD
PARTITION p_new "
+ + "VALUES [('2020-01-01'), ('2020-01-02'))");
+
+ try {
+ setDynamicPartitionDropProtection(true);
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+ Assertions.assertTrue(table.getPartitionNames().contains("p_old"));
+ Assertions.assertTrue(table.getPartitionNames().contains("p_new"));
+
+ setDynamicPartitionDropProtection(false);
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+
Assertions.assertFalse(table.getPartitionNames().contains("p_old"));
+ Assertions.assertTrue(table.getPartitionNames().contains("p_new"));
+ } finally {
+ setDynamicPartitionDropProtection(false);
+ }
+ }
+
@Test
public void testNormal() throws Exception {
String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_normal` (\n"
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
index 455e9801e9a..f584d59572a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -87,6 +87,32 @@ public class VariableMgrTest extends TestWithFeService {
Assertions.assertThrows(DdlException.class, () ->
VariableMgr.setVar(null, setVar));
}
+ @Test
+ public void testDynamicPartitionDropProtectionIsGlobalOnly() throws
Exception {
+ try {
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
"false");
+
Assertions.assertFalse(VariableMgr.isDynamicPartitionDropProtectionEnabled());
+
+ SetOptionsCommand sessionStmt = (SetOptionsCommand)
UtFrameUtils.parseStmt(
+ "set enable_dynamic_partition_drop_protection=true",
connectContext);
+ Assertions.assertThrows(DdlException.class, () ->
sessionStmt.run(connectContext, null));
+
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
"true");
+
Assertions.assertTrue(VariableMgr.isDynamicPartitionDropProtectionEnabled());
+
Assertions.assertTrue(VariableMgr.newSessionVariable().isEnableDynamicPartitionDropProtection());
+
+ SetOptionsCommand globalStmt = (SetOptionsCommand)
UtFrameUtils.parseStmt(
+ "set global
enable_dynamic_partition_drop_protection=false", connectContext);
+ globalStmt.run(connectContext, null);
+
Assertions.assertFalse(VariableMgr.isDynamicPartitionDropProtectionEnabled());
+ } finally {
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
"false");
+ }
+ }
+
@Test
public void testAuthenticationPolicyIsTheReadOnlyMysqlDefault() throws
Exception {
SessionVariable sessionVariable = new SessionVariable();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
