This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8526b9ffbe3 [imporvement](table property) support for alter table
property disable_auto_compaction (#27961)
8526b9ffbe3 is described below
commit 8526b9ffbe31d226749ff03505f91c0ef8c6873c
Author: xueweizhang <[email protected]>
AuthorDate: Thu Dec 7 15:08:39 2023 +0800
[imporvement](table property) support for alter table property
disable_auto_compaction (#27961)
in some case, some tablets may cause coredump or OOM when compaction, and
it is necessary to manually close the compaction of a specific table by
'disable_auto_compaction' to make be service available
This commit allow modify disable_auto_compaction table property in schema
change.
---------
Signed-off-by: nextdreamblue <[email protected]>
---
be/src/agent/task_worker_pool.cpp | 12 ++++++++++++
.../src/main/java/org/apache/doris/alter/Alter.java | 2 ++
.../org/apache/doris/alter/SchemaChangeHandler.java | 20 +++++++++++++++-----
.../doris/analysis/ModifyTablePropertiesClause.java | 10 ++++++++++
.../src/main/java/org/apache/doris/catalog/Env.java | 1 +
.../java/org/apache/doris/catalog/TableProperty.java | 1 +
.../apache/doris/task/UpdateTabletMetaInfoTask.java | 8 +++++++-
gensrc/thrift/AgentService.thrift | 3 ++-
.../schema_change/test_alter_table_property.groovy | 12 +++++++++++-
9 files changed, 61 insertions(+), 8 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 7139fefcde1..5bb3fb8ad79 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -770,6 +770,18 @@ void update_tablet_meta_callback(StorageEngine& engine,
const TAgentTaskRequest&
tablet_meta_info.enable_single_replica_compaction);
need_to_save = true;
}
+ if (tablet_meta_info.__isset.disable_auto_compaction) {
+ std::shared_lock rlock(tablet->get_header_lock());
+
tablet->tablet_meta()->mutable_tablet_schema()->set_disable_auto_compaction(
+ tablet_meta_info.disable_auto_compaction);
+ for (auto& rowset_meta :
tablet->tablet_meta()->all_mutable_rs_metas()) {
+ rowset_meta->tablet_schema()->set_disable_auto_compaction(
+ tablet_meta_info.disable_auto_compaction);
+ }
+ tablet->tablet_schema_unlocked()->set_disable_auto_compaction(
+ tablet_meta_info.disable_auto_compaction);
+ need_to_save = true;
+ }
if (tablet_meta_info.__isset.skip_write_index_on_load) {
std::shared_lock rlock(tablet->get_header_lock());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 1cc679740e1..5568248e8d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -504,6 +504,8 @@ public class Alter {
.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
+ || properties
+
.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD));
((SchemaChangeHandler)
schemaChangeHandler).updateTableProperties(db, tableName, properties);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index e77e7f11d22..dc238044ebe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2194,6 +2194,7 @@ public class SchemaChangeHandler extends AlterHandler {
if (isInMemory < 0 && storagePolicyId < 0 && compactionPolicy == null
&& timeSeriesCompactionConfig.isEmpty()
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
+ &&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
LOG.info("Properties already up-to-date");
@@ -2206,6 +2207,12 @@ public class SchemaChangeHandler extends AlterHandler {
enableSingleCompaction = Boolean.parseBoolean(singleCompaction) ?
1 : 0;
}
+ String disableAutoCompactionBoolean =
properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION);
+ int disableAutoCompaction = -1; // < 0 means don't update
+ if (disableAutoCompactionBoolean != null) {
+ disableAutoCompaction =
Boolean.parseBoolean(disableAutoCompactionBoolean) ? 1 : 0;
+ }
+
String skipWriteIndexOnLoad =
properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD);
int skip = -1; // < 0 means don't update
if (skipWriteIndexOnLoad != null) {
@@ -2214,7 +2221,8 @@ public class SchemaChangeHandler extends AlterHandler {
for (Partition partition : partitions) {
updatePartitionProperties(db, olapTable.getName(),
partition.getName(), storagePolicyId, isInMemory,
- null, compactionPolicy,
timeSeriesCompactionConfig, enableSingleCompaction, skip);
+ null, compactionPolicy,
timeSeriesCompactionConfig, enableSingleCompaction, skip,
+ disableAutoCompaction);
}
olapTable.writeLockOrDdlException();
@@ -2257,7 +2265,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (String partitionName : partitionNames) {
try {
updatePartitionProperties(db, olapTable.getName(),
partitionName, storagePolicyId,
-
isInMemory, null, null, null, -1, -1);
+
isInMemory, null, null, null, -1, -1, -1);
} catch (Exception e) {
String errMsg = "Failed to update partition[" + partitionName
+ "]'s 'in_memory' property. "
+ "The reason is [" + e.getMessage() + "]";
@@ -2273,7 +2281,8 @@ public class SchemaChangeHandler extends AlterHandler {
public void updatePartitionProperties(Database db, String tableName,
String partitionName, long storagePolicyId,
int isInMemory, BinlogConfig
binlogConfig, String compactionPolicy,
Map<String, Long>
timeSeriesCompactionConfig,
- int enableSingleCompaction, int
skipWriteIndexOnLoad) throws UserException {
+ int enableSingleCompaction, int
skipWriteIndexOnLoad,
+ int disableAutoCompaction) throws
UserException {
// be id -> <tablet id,schemaHash>
Map<Long, Set<Pair<Long, Integer>>> beIdToTabletIdWithHash =
Maps.newHashMap();
OlapTable olapTable = (OlapTable)
db.getTableOrMetaException(tableName, Table.TableType.OLAP);
@@ -2306,7 +2315,8 @@ public class SchemaChangeHandler extends AlterHandler {
countDownLatch.addMark(kv.getKey(), kv.getValue());
UpdateTabletMetaInfoTask task = new
UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), isInMemory,
storagePolicyId, binlogConfig,
countDownLatch, compactionPolicy,
- timeSeriesCompactionConfig,
enableSingleCompaction, skipWriteIndexOnLoad);
+ timeSeriesCompactionConfig,
enableSingleCompaction, skipWriteIndexOnLoad,
+ disableAutoCompaction);
batchTask.addTask(task);
}
if (!FeConstants.runningUnitTest) {
@@ -3011,7 +3021,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Partition partition : partitions) {
updatePartitionProperties(db, olapTable.getName(),
partition.getName(), -1, -1,
- newBinlogConfig, null, null,
-1, -1);
+ newBinlogConfig, null, null,
-1, -1, -1);
}
olapTable.writeLockOrDdlException();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index 0cbb1d190ee..a23a73df157 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -226,6 +226,16 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)) {
+ if
(!properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("true")
+ && !properties.get(PropertyAnalyzer
+
.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("false")) {
+ throw new AnalysisException(
+ "Property "
+ + PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION
+ " should be set to true or false");
+ }
+ this.needTableStable = false;
+ this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)) {
long groupCommitIntervalMs;
String groupCommitIntervalMsStr =
properties.get(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 61379c1b28e..38069eaa5fd 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4698,6 +4698,7 @@ public class Env {
.buildTimeSeriesCompactionFileCountThreshold()
.buildTimeSeriesCompactionTimeThresholdSeconds()
.buildSkipWriteIndexOnLoad()
+ .buildDisableAutoCompaction()
.buildEnableSingleReplicaCompaction();
// need to update partition info meta
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 9f299befadb..dd44dd38680 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -131,6 +131,7 @@ public class TableProperty implements Writable {
buildTimeSeriesCompactionTimeThresholdSeconds();
buildSkipWriteIndexOnLoad();
buildEnableSingleReplicaCompaction();
+ buildDisableAutoCompaction();
break;
default:
break;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
index 4eb19a5bfa0..af5e45a8256 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
@@ -53,6 +53,7 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
// < 0 means not to update property, > 0 means true, == 0 means false
private int enableSingleReplicaCompaction = -1;
private int skipWriteIndexOnLoad = -1;
+ private int disableAutoCompaction = -1;
public UpdateTabletMetaInfoTask(long backendId, Set<Pair<Long, Integer>>
tableIdWithSchemaHash) {
super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO,
@@ -87,12 +88,14 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
String compactionPolicy,
Map<String, Long>
timeSeriesCompactionConfig,
int enableSingleReplicaCompaction,
- int skipWriteIndexOnLoad) {
+ int skipWriteIndexOnLoad,
+ int disableAutoCompaction) {
this(backendId, tableIdWithSchemaHash, inMemory, storagePolicyId,
binlogConfig, latch);
this.compactionPolicy = compactionPolicy;
this.timeSeriesCompactionConfig = timeSeriesCompactionConfig;
this.enableSingleReplicaCompaction = enableSingleReplicaCompaction;
this.skipWriteIndexOnLoad = skipWriteIndexOnLoad;
+ this.disableAutoCompaction = disableAutoCompaction;
}
public void countDownLatch(long backendId, Set<Pair<Long, Integer>>
tablets) {
@@ -159,6 +162,9 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
if (skipWriteIndexOnLoad >= 0) {
metaInfo.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad > 0);
}
+ if (disableAutoCompaction >= 0) {
+ metaInfo.setDisableAutoCompaction(disableAutoCompaction >
0);
+ }
updateTabletMetaInfoReq.addToTabletMetaInfos(metaInfo);
}
} else {
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 79bb014a901..c83660e9b9b 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -418,6 +418,7 @@ struct TTabletMetaInfo {
13: optional i64 time_series_compaction_time_threshold_seconds
14: optional bool enable_single_replica_compaction
15: optional bool skip_write_index_on_load
+ 16: optional bool disable_auto_compaction
}
struct TUpdateTabletMetaInfoReq {
@@ -507,4 +508,4 @@ struct TTopicUpdate {
struct TAgentPublishRequest {
1: required TAgentServiceVersion protocol_version
2: required list<TTopicUpdate> updates
-}
\ No newline at end of file
+}
diff --git
a/regression-test/suites/schema_change/test_alter_table_property.groovy
b/regression-test/suites/schema_change/test_alter_table_property.groovy
index 6442b9c00e9..1b9b5f810c7 100644
--- a/regression-test/suites/schema_change/test_alter_table_property.groovy
+++ b/regression-test/suites/schema_change/test_alter_table_property.groovy
@@ -46,6 +46,16 @@ suite("test_alter_table_property") {
logger.info("${showResult2}")
assertTrue(showResult2.toString().containsIgnoreCase('"enable_single_replica_compaction"
= "true"'))
+
assertTrue(showResult1.toString().containsIgnoreCase('"disable_auto_compaction"
= "false"'))
+ sql """
+ alter table ${tableName} set ("disable_auto_compaction" = "true")
+ """
+ sql """sync"""
+
+ def showResult3 = sql """show create table ${tableName}"""
+ logger.info("${showResult3}")
+
assertTrue(showResult3.toString().containsIgnoreCase('"disable_auto_compaction"
= "true"'))
+
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """sync"""
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
