This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7f604276c7c [improvement](truncate table) truncate skip empty
partition (#28229)
7f604276c7c is described below
commit 7f604276c7c5988e4d2706435eeb68596b642184
Author: yujun <[email protected]>
AuthorDate: Tue Dec 12 21:12:31 2023 +0800
[improvement](truncate table) truncate skip empty partition (#28229)
---
.../apache/doris/datasource/InternalCatalog.java | 23 ++++++++---
.../apache/doris/catalog/TruncateTableTest.java | 10 +++--
.../data/ddl_p0/test_truncate_table.out | 12 ++++++
.../suites/ddl_p0/test_truncate_table.groovy | 46 ++++++++++++++--------
4 files changed, 67 insertions(+), 24 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index d4943e6cb80..9b96dfa4c31 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2922,19 +2922,31 @@ public class InternalCatalog implements
CatalogIf<Database> {
if (partition == null) {
throw new DdlException("Partition " + partName + "
does not exist");
}
+ // If need absolutely correct, should check running txn
here.
+ // But if the txn is in prepare state, cann't known which
partitions had load data.
+ if (!partition.hasData()) {
+ continue;
+ }
origPartitions.put(partName, partition.getId());
partitionsDistributionInfo.put(partition.getId(),
partition.getDistributionInfo());
rowsToTruncate += partition.getBaseIndex().getRowCount();
}
} else {
for (Partition partition : olapTable.getPartitions()) {
+ // If need absolutely correct, should check running txn
here.
+ // But if the txn is in prepare state, cann't known which
partitions had load data.
+ if (!partition.hasData()) {
+ continue;
+ }
origPartitions.put(partition.getName(), partition.getId());
partitionsDistributionInfo.put(partition.getId(),
partition.getDistributionInfo());
}
}
- // if table currently has no partitions, this sql like empty
command and do nothing, should return directly
- // at the same time, it will avoid throwing IllegalStateException
when `bufferSize` equals zero
- if (origPartitions.isEmpty()) {
+ // if table currently has no partitions, this sql like empty
command and do nothing, should return directly.
+ // but if truncate whole table, the temporary partitions also need
drop
+ if (origPartitions.isEmpty() && (!truncateEntireTable ||
olapTable.getTempPartitions().isEmpty())) {
+ LOG.info("finished to truncate table {}, no partition contains
data, do nothing",
+ tblRef.getName().toSql());
return;
}
copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(),
IndexExtState.VISIBLE, false);
@@ -2948,8 +2960,6 @@ public class InternalCatalog implements
CatalogIf<Database> {
List<Partition> newPartitions = Lists.newArrayList();
// tabletIdSet to save all newly created tablet ids.
Set<Long> tabletIdSet = Sets.newHashSet();
- long bufferSize =
IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl,
origPartitions.values());
- IdGeneratorBuffer idGeneratorBuffer =
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
Map<Integer, Integer> clusterKeyMap = new TreeMap<>();
for (int i = 0; i < olapTable.getBaseSchema().size(); i++) {
Column column = olapTable.getBaseSchema().get(i);
@@ -2959,6 +2969,9 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
List<Integer> clusterKeyIdxes =
clusterKeyMap.values().stream().collect(Collectors.toList());
try {
+ long bufferSize =
IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl,
origPartitions.values());
+ IdGeneratorBuffer idGeneratorBuffer =
+ origPartitions.isEmpty() ? null :
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
for (Map.Entry<String, Long> entry : origPartitions.entrySet()) {
// the new partition must use new id
// If we still use the old partition id, the behavior of
current load jobs on this partition
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
index c13730f32d3..3beccd3bfe6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
@@ -90,7 +90,10 @@ public class TruncateTableTest {
long p20211006Id = tbl.getPartition("p20211006").getId();
long p20211007Id = tbl.getPartition("P20211007").getId();
long p20211008Id = tbl.getPartition("P20211008").getId();
- // truncate p20211008(real name is P20211008)
+ // truncate P20211008(real name is P20211008)
+ Partition p20211008 = tbl.getPartition("P20211008");
+ p20211008.updateVisibleVersion(2L);
+ p20211008.setNextVersion(p20211008.getVisibleVersion() + 1);
String truncateStr = "TRUNCATE TABLE test.case_sensitive_table
PARTITION P20211008; \n";
TruncateTableStmt truncateTableStmt
= (TruncateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
@@ -101,10 +104,11 @@ public class TruncateTableTest {
truncateTableStmt = (TruncateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
Env.getCurrentEnv().truncateTable(truncateTableStmt);
Assert.assertEquals(3, tbl.getPartitionInfo().idToDataProperty.size());
- Assert.assertNotEquals(p20211007Id,
tbl.getPartition("P20211007").getId());
Assert.assertEquals(p20211006Id,
tbl.getPartition("p20211006").getId());
+ Assert.assertEquals(p20211007Id,
tbl.getPartition("P20211007").getId());
Assert.assertNotNull(tbl.getPartition("p20211006"));
- Assert.assertNotNull(tbl.getPartition("p20211006"));
+ Assert.assertNotNull(tbl.getPartition("P20211007"));
+ Assert.assertNotNull(tbl.getPartition("P20211008"));
}
@Test
diff --git a/regression-test/data/ddl_p0/test_truncate_table.out
b/regression-test/data/ddl_p0/test_truncate_table.out
new file mode 100644
index 00000000000..95ed72f34af
--- /dev/null
+++ b/regression-test/data/ddl_p0/test_truncate_table.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_1 --
+2020-01-01 1.00 a 1
+2020-03-10 1.00 a 1
+
+-- !select_2 --
+
+-- !select_3 --
+2020-02-10 1.00 a 1
+
+-- !select_4 --
+
diff --git a/regression-test/suites/ddl_p0/test_truncate_table.groovy
b/regression-test/suites/ddl_p0/test_truncate_table.groovy
index 54487b23f00..7a79d04c70e 100644
--- a/regression-test/suites/ddl_p0/test_truncate_table.groovy
+++ b/regression-test/suites/ddl_p0/test_truncate_table.groovy
@@ -40,24 +40,38 @@ suite("test_truncate_table") {
"replication_num" = "1"
);
"""
- List<List<Object>> result = sql "show partitions from ${testTable}"
- logger.info("${result}")
- assertEquals(result.size(), 3)
- assertEquals(result.get(0).get(1), "p1")
+
+ def getPartitionIds = { ->
+ def result = sql_return_maparray("show partitions from ${testTable}")
+ return result.collectEntries { [it.PartitionName, it.PartitionId as
long] }
+ }
+
+ def partitionIds1 = getPartitionIds()
+ assertEquals(["p1", "p2", "p3"].toSet(), partitionIds1.keySet())
+
+ sql "insert into ${testTable} values ('2020-01-01', 1.0, 'a', 1)"
+ sql "insert into ${testTable} values ('2020-03-10', 1.0, 'a', 1)"
+ order_qt_select_1 "SELECT * FROM ${testTable}"
sql """truncate table ${testTable};"""
- result = sql "show partitions from ${testTable}"
- logger.info("${result}")
- assertEquals(result.size(), 3)
- assertEquals(result.get(0).get(1), "p1")
-
- sql """truncate table ${testTable} partitions (p1, p1);"""
-
- result = sql "show partitions from ${testTable}"
- logger.info("${result}")
- assertEquals(result.size(), 3)
- assertEquals(result.get(0).get(1), "p1")
-
+ def partitionIds2 = getPartitionIds()
+ assertEquals(["p1", "p2", "p3"].toSet(), partitionIds2.keySet())
+ assertNotEquals(partitionIds1.get("p1"), partitionIds2.get("p1"))
+ assertEquals(partitionIds1.get("p2"), partitionIds2.get("p2"))
+ assertNotEquals(partitionIds1.get("p3"), partitionIds2.get("p3"))
+ order_qt_select_2 "SELECT * FROM ${testTable}"
+
+ sql "insert into ${testTable} values ('2020-02-10', 1.0, 'a', 1)"
+ order_qt_select_3 "SELECT * FROM ${testTable}"
+ sql """truncate table ${testTable} partitions (p1, p2);"""
+ order_qt_select_4 "SELECT * FROM ${testTable}"
+
+ def partitionIds3 = getPartitionIds()
+ assertEquals(["p1", "p2", "p3"].toSet(), partitionIds3.keySet())
+ assertEquals(partitionIds2.get("p1"), partitionIds3.get("p1"))
+ assertNotEquals(partitionIds2.get("p2"), partitionIds3.get("p2"))
+ assertEquals(partitionIds2.get("p3"), partitionIds3.get("p3"))
+
sql "DROP TABLE IF EXISTS ${testTable}"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
