This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 3f4002e778b branch-4.1: [feature](index) Support partition DROP INDEX
with conflict guards (#70357)
3f4002e778b is described below
commit 3f4002e778bd4bdac135b349f349e5ed4d7a5093
Author: Jack <[email protected]>
AuthorDate: Fri Nov 6 21:52:55 2026 +0800
branch-4.1: [feature](index) Support partition DROP INDEX with conflict
guards (#70357)
### What problem does this PR solve?
Issue Number: No separate issue is linked by the source PRs.
Related PRs: [#62181](https://github.com/apache/doris/pull/62181),
[#69442](https://github.com/apache/doris/pull/69442),
[#69443](https://github.com/apache/doris/pull/69443)
Backport #62181 with the two follow-up correctness fixes #69442 and
#69443. Partition DROP INDEX deletes physical inverted-index files from
selected partitions while retaining the table index definition. Mixed
ALTER work and unfinished BUILD/DROP jobs on the same index are rejected
before metadata changes. The older target needs a DropIndexClause
adapter and a minimal early full-table conflict check.
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 5 +-
.../org/apache/doris/alter/IndexChangeJob.java | 14 +-
.../apache/doris/alter/SchemaChangeHandler.java | 169 ++++++++++---
.../org/apache/doris/analysis/DropIndexClause.java | 36 +++
.../doris/nereids/parser/LogicalPlanBuilder.java | 16 +-
.../trees/plans/commands/info/DropIndexOp.java | 44 +++-
.../org/apache/doris/alter/IndexChangeJobTest.java | 131 ++++++++++
.../doris/alter/SchemaChangeHandlerTest.java | 178 ++++++++++++++
.../test_drop_index_on_partition.groovy | 269 +++++++++++++++++++++
9 files changed, 813 insertions(+), 49 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 5d9325534f7..16cb8d92e00 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -363,7 +363,8 @@ supportedDropStatement
| DROP (DATABASE | SCHEMA) (IF EXISTS)? name=multipartIdentifier FORCE?
#dropDatabase
| DROP statementScope? FUNCTION (IF EXISTS)?
functionIdentifier LEFT_PAREN functionArguments? RIGHT_PAREN
#dropFunction
- | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier
#dropIndex
+ | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier
+ partitionSpec?
#dropIndex
| DROP RESOURCE (IF EXISTS)? name=identifierOrText
#dropResource
| DROP ROW POLICY (IF EXISTS)? policyName=identifier
ON tableName=multipartIdentifier
@@ -793,7 +794,7 @@ alterTableClause
| RENAME PARTITION name=identifier newName=identifier
#renamePartitionClause
| RENAME COLUMN name=qualifiedName TO? newName=identifier
#renameColumnClause
| ADD indexDef
#addIndexClause
- | DROP INDEX (IF EXISTS)? name=identifier
#dropIndexClause
+ | DROP INDEX (IF EXISTS)? name=identifier partitionSpec?
#dropIndexClause
| ENABLE FEATURE name=STRING_LITERAL (WITH properties=propertyClause)?
#enableFeatureClause
| MODIFY DISTRIBUTION (DISTRIBUTED BY (HASH hashKeys=identifierList |
RANDOM)
(BUCKETS (INTEGER_VALUE | autoBucket=AUTO))?)?
#modifyDistributionClause
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
index f25f82cda90..facb618b372 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
@@ -173,13 +173,13 @@ public class IndexChangeJob implements Writable {
this.alterInvertedIndexes = alterInvertedIndexes;
}
- public boolean hasSameAlterInvertedIndex(boolean isDropOp, List<Index>
inputAlterInvertedIndexes) {
- if (this.isDropOp == isDropOp) {
- for (Index inputIndex : inputAlterInvertedIndexes) {
- for (Index existIndex : this.alterInvertedIndexes) {
- if (inputIndex.getIndexId() == existIndex.getIndexId()) {
- return true;
- }
+ // Whether this job builds or drops any of the given indexes, in either
direction: jobs of one index are
+ // not ordered, so a build running after a drop would add the dropped
index back, and the other way round.
+ public boolean hasAlterInvertedIndex(List<Index>
inputAlterInvertedIndexes) {
+ for (Index inputIndex : inputAlterInvertedIndexes) {
+ for (Index existIndex : this.alterInvertedIndexes) {
+ if (inputIndex.getIndexId() == existIndex.getIndexId()) {
+ return true;
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 921da654421..4d238e871c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1999,6 +1999,15 @@ public class SchemaChangeHandler extends AlterHandler {
Map<Long, Set<String>> indexOnPartitions = new HashMap<>();
boolean isDropIndex = false;
Map<String, String> propertyMap = new HashMap<>();
+ checkLightDropIndexJobConflicts(db, olapTable, alterClauses,
enableAddIndexForNewData);
+ // Partition index jobs cannot apply other schema changes or
independent partition scopes.
+ if (alterClauses.size() > 1 && alterClauses.stream().anyMatch(
+ alterClause -> alterClause instanceof DropIndexClause
+ && ((DropIndexClause)
alterClause).hasPartitionSpec())) {
+ throw new DdlException("DROP INDEX on partitions can not be
combined with other clauses in one"
+ + " ALTER statement, please run them in separate
statements.");
+ }
+
for (AlterClause alterClause : alterClauses) {
Map<String, String> properties = alterClause.getProperties();
// alter table properties
@@ -2179,37 +2188,72 @@ public class SchemaChangeHandler extends AlterHandler {
buildIndexChange = true;
lightSchemaChange = false;
} else if (alterClause instanceof DropIndexClause) {
- if (processDropIndex((DropIndexClause) alterClause,
olapTable, newIndexes)) {
- return;
- }
- lightSchemaChange = false;
-
DropIndexClause dropIndexClause = (DropIndexClause)
alterClause;
- List<Index> existedIndexes = olapTable.getIndexes();
- Index found = null;
- for (Index existedIdx : existedIndexes) {
- if
(existedIdx.getIndexName().equalsIgnoreCase(dropIndexClause.getIndexName())) {
- found = existedIdx;
- break;
+
+ if (dropIndexClause.hasPartitionSpec()) {
+ // Delete the physical index files and keep the
table-level index definition.
+ List<Index> existedIndexes = olapTable.getIndexes();
+ Index found = null;
+ for (Index existedIdx : existedIndexes) {
+ if
(existedIdx.getIndexName().equalsIgnoreCase(dropIndexClause.getIndexName())) {
+ found = existedIdx;
+ break;
+ }
}
- }
- // for inverted index, light schema change is supported in
both cloud and local mode;
- // for ngram index, light schema change is supported only
in cloud mode;
- boolean supportLightIndexChange = false;
- if (Config.isCloudMode()) {
- if (enableAddIndexForNewData) {
- supportLightIndexChange = (
- found.getIndexType() == IndexType.NGRAM_BF
- || found.getIndexType() ==
IndexDef.IndexType.INVERTED);
+ if (found == null) {
+ if (dropIndexClause.isSetIfExists()) {
+ LOG.info("drop index[{}] which does not exist
on table[{}]",
+ dropIndexClause.getIndexName(),
olapTable.getName());
+ return;
+ }
+ throw new DdlException("index " +
dropIndexClause.getIndexName() + " does not exist");
}
- } else {
- supportLightIndexChange = found.getIndexType() ==
IndexDef.IndexType.INVERTED
- || found.getIndexType() ==
IndexDef.IndexType.ANN;
- }
- if (found != null && supportLightIndexChange) {
+ if (found.getIndexType() != IndexType.INVERTED) {
+ throw new DdlException(
+ "Only inverted index supports DROP INDEX
ON PARTITION");
+ }
+ if (!olapTable.isPartitionedTable()) {
+ throw new DdlException("table " +
olapTable.getName()
+ + " is not partitioned, cannot drop index
with partitions");
+ }
+ Set<String> specifiedPartitions = new
HashSet<>(dropIndexClause.getPartitionNames());
+ for (String partName : specifiedPartitions) {
+ if (olapTable.getPartition(partName) == null) {
+ throw new DdlException("partition " + partName
+ " does not exist");
+ }
+ }
+
+ if (!Config.enable_light_index_change) {
+ throw new DdlException(
+ "DROP INDEX ON PARTITION requires
enable_light_index_change=true");
+ }
+ checkNoIndexChangeJobOnPartitions(db, olapTable,
+ ImmutableList.of(found), specifiedPartitions);
+
alterIndexes.add(found);
+ indexOnPartitions.put(found.getIndexId(),
specifiedPartitions);
isDropIndex = true;
- lightIndexChange = true;
+ buildIndexChange = true;
+ lightSchemaChange = false;
+ } else {
+ if (processDropIndex(dropIndexClause, olapTable,
newIndexes)) {
+ return;
+ }
+ lightSchemaChange = false;
+
+ List<Index> existedIndexes = olapTable.getIndexes();
+ Index found = null;
+ for (Index existedIdx : existedIndexes) {
+ if
(existedIdx.getIndexName().equalsIgnoreCase(dropIndexClause.getIndexName())) {
+ found = existedIdx;
+ break;
+ }
+ }
+ if (found != null && isLightDropIndexSupported(found,
enableAddIndexForNewData)) {
+ alterIndexes.add(found);
+ isDropIndex = true;
+ lightIndexChange = true;
+ }
}
} else {
Preconditions.checkState(false);
@@ -2239,7 +2283,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
if (Config.enable_light_index_change) {
buildOrDeleteTableInvertedIndices(db, olapTable,
indexSchemaMap,
- alterIndexes, indexOnPartitions, false);
+ alterIndexes, indexOnPartitions, isDropIndex);
}
} else {
createJob(rawSql, db.getId(), olapTable, indexSchemaMap,
propertyMap, newIndexes);
@@ -3411,7 +3455,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
if (hasIndexChangeJobOnPartition(originIndexId,
db.getId(), olapTable.getId(),
- partitionName, alterIndexes, isDropOp)) {
+ partitionName, alterIndexes)) {
throw new DdlException("partition " + partitionName +
" has been built specified index."
+ " please check your build
stmt.");
}
@@ -3435,25 +3479,76 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
+ private static boolean isLightDropIndexSupported(Index index, boolean
enableAddIndexForNewData) {
+ if (Config.isCloudMode()) {
+ return enableAddIndexForNewData
+ && (index.getIndexType() == IndexType.NGRAM_BF
+ || index.getIndexType() == IndexType.INVERTED);
+ }
+ return index.getIndexType() == IndexType.INVERTED ||
index.getIndexType() == IndexType.ANN;
+ }
+
+ // Reject conflicting jobs before a light full-table drop changes the
index metadata.
+ private void checkLightDropIndexJobConflicts(Database db, OlapTable
olapTable, List<AlterClause> alterClauses,
+ boolean enableAddIndexForNewData) throws DdlException {
+ if (!Config.enable_light_index_change) {
+ return;
+ }
+ List<Index> droppedIndexes = new ArrayList<>();
+ for (AlterClause alterClause : alterClauses) {
+ if (!(alterClause instanceof DropIndexClause)
+ || ((DropIndexClause) alterClause).hasPartitionSpec()) {
+ continue;
+ }
+ String indexName = ((DropIndexClause) alterClause).getIndexName();
+ olapTable.getIndexes().stream()
+ .filter(index ->
index.getIndexName().equalsIgnoreCase(indexName))
+ .filter(index -> isLightDropIndexSupported(index,
enableAddIndexForNewData))
+ .findFirst()
+ .ifPresent(droppedIndexes::add);
+ }
+ if (droppedIndexes.isEmpty()) {
+ return;
+ }
+ checkNoIndexChangeJobOnPartitions(db, olapTable, droppedIndexes,
olapTable.getPartitionNames());
+ }
+
+ private void checkNoIndexChangeJobOnPartitions(Database db, OlapTable
olapTable, List<Index> alterIndexes,
+ Set<String> partitionNames) throws DdlException {
+ for (IndexChangeJob indexChangeJob :
ImmutableList.copyOf(indexChangeJobs.values())) {
+ if (partitionNames.contains(indexChangeJob.getPartitionName())
+ && isConflictingIndexChangeJob(indexChangeJob,
olapTable.getBaseIndexId(), db.getId(),
+ olapTable.getId(), alterIndexes)) {
+ throw new DdlException("partition " +
indexChangeJob.getPartitionName() + " of table "
+ + olapTable.getName() + " has an unfinished index
change job, please retry after it finishes.");
+ }
+ }
+ }
+
public boolean hasIndexChangeJobOnPartition(
long originIndexId, long dbId, long tableId, String partitionName,
- List<Index> alterIndexes, boolean isDrop) {
+ List<Index> alterIndexes) {
// TODO: this is temporary methods
for (IndexChangeJob indexChangeJob :
ImmutableList.copyOf(indexChangeJobs.values())) {
- if (indexChangeJob.getOriginIndexId() == originIndexId
- && indexChangeJob.getDbId() == dbId
- && indexChangeJob.getTableId() == tableId
- && indexChangeJob.getPartitionName().equals(partitionName)
- && ((Config.isNotCloudMode() &&
indexChangeJob.hasSameAlterInvertedIndex(isDrop, alterIndexes))
- || Config.isCloudMode())
- && !indexChangeJob.isDone()) {
- // if JobState is done (CANCELLED or FINISHED), also allow
user to create job again
+ if (indexChangeJob.getPartitionName().equals(partitionName)
+ && isConflictingIndexChangeJob(indexChangeJob,
originIndexId, dbId, tableId, alterIndexes)) {
return true;
}
}
return false;
}
+ private static boolean isConflictingIndexChangeJob(IndexChangeJob
indexChangeJob, long originIndexId,
+ long dbId, long tableId, List<Index> alterIndexes) {
+ // if JobState is done (CANCELLED or FINISHED), also allow user to
create job again
+ return indexChangeJob.getOriginIndexId() == originIndexId
+ && indexChangeJob.getDbId() == dbId
+ && indexChangeJob.getTableId() == tableId
+ && ((Config.isNotCloudMode() &&
indexChangeJob.hasAlterInvertedIndex(alterIndexes))
+ || Config.isCloudMode())
+ && !indexChangeJob.isDone();
+ }
+
// schema change job is not allowed when index change job is running
boolean hasIndexChangeJobOnTable(long dbId, long tableId) {
for (IndexChangeJob indexChangeJob :
ImmutableList.copyOf(indexChangeJobs.values())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropIndexClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropIndexClause.java
index 822d0c875fa..51910d3cb19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropIndexClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropIndexClause.java
@@ -20,9 +20,12 @@ package org.apache.doris.analysis;
import org.apache.doris.alter.AlterOpType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
+import org.apache.doris.info.PartitionNamesInfo;
import org.apache.commons.lang3.StringUtils;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
public class DropIndexClause extends AlterTableClause {
@@ -32,12 +35,20 @@ public class DropIndexClause extends AlterTableClause {
private boolean alter;
+ private final PartitionNamesInfo partitionNamesInfo;
+
public DropIndexClause(String indexName, boolean ifExists, TableName
tableName, boolean alter) {
+ this(indexName, ifExists, tableName, alter, null);
+ }
+
+ public DropIndexClause(String indexName, boolean ifExists, TableName
tableName, boolean alter,
+ PartitionNamesInfo partitionNamesInfo) {
super(AlterOpType.SCHEMA_CHANGE);
this.indexName = indexName;
this.ifExists = ifExists;
this.tableName = tableName;
this.alter = alter;
+ this.partitionNamesInfo = partitionNamesInfo;
}
public String getIndexName() {
@@ -56,6 +67,16 @@ public class DropIndexClause extends AlterTableClause {
return alter;
}
+ public boolean hasPartitionSpec() {
+ return partitionNamesInfo != null;
+ }
+
+ public List<String> getPartitionNames() {
+ return partitionNamesInfo == null
+ ? Collections.emptyList()
+ : partitionNamesInfo.getPartitionNames();
+ }
+
@Override
public Map<String, String> getProperties() {
return null;
@@ -66,6 +87,18 @@ public class DropIndexClause extends AlterTableClause {
if (StringUtils.isEmpty(indexName)) {
throw new AnalysisException("index name is excepted");
}
+ if (partitionNamesInfo != null) {
+ if (partitionNamesInfo.isTemp()) {
+ throw new AnalysisException(
+ "DROP INDEX ON PARTITION does not support temporary
partitions");
+ }
+ if (partitionNamesInfo.getPartitionNames() == null
+ || partitionNamesInfo.getPartitionNames().isEmpty()) {
+ throw new AnalysisException(
+ "DROP INDEX ON PARTITION requires explicit partition
names, "
+ + "PARTITIONS (*) is not supported");
+ }
+ }
}
@Override
@@ -89,6 +122,9 @@ public class DropIndexClause extends AlterTableClause {
if (!alter) {
stringBuilder.append(" ON ").append(tableName.toSql());
}
+ if (partitionNamesInfo != null) {
+ stringBuilder.append(" ").append(partitionNamesInfo.toSql());
+ }
return stringBuilder.toString();
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 3606f10fc7b..413434741de 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -6567,14 +6567,26 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
public Command visitDropIndex(DropIndexContext ctx) {
String name = ctx.name.getText();
TableNameInfo tableName = new
TableNameInfo(visitMultipartIdentifier(ctx.tableName));
+ PartitionNamesInfo partitionNamesInfo = null;
+ if (ctx.partitionSpec() != null) {
+ Pair<Boolean, List<String>> partitionSpec =
visitPartitionSpec(ctx.partitionSpec());
+ partitionNamesInfo = new PartitionNamesInfo(partitionSpec.first,
partitionSpec.second);
+ }
List<AlterTableOp> alterTableOps = Lists
- .newArrayList(new DropIndexOp(name, ctx.EXISTS() != null,
tableName, false));
+ .newArrayList(new DropIndexOp(name, ctx.EXISTS() != null,
tableName, false,
+ partitionNamesInfo));
return new AlterTableCommand(tableName, alterTableOps);
}
@Override
public AlterTableOp visitDropIndexClause(DropIndexClauseContext ctx) {
- return new DropIndexOp(ctx.name.getText(), ctx.EXISTS() != null, null,
true);
+ PartitionNamesInfo partitionNamesInfo = null;
+ if (ctx.partitionSpec() != null) {
+ Pair<Boolean, List<String>> partitionSpec =
visitPartitionSpec(ctx.partitionSpec());
+ partitionNamesInfo = new PartitionNamesInfo(partitionSpec.first,
partitionSpec.second);
+ }
+ return new DropIndexOp(ctx.name.getText(), ctx.EXISTS() != null, null,
true,
+ partitionNamesInfo);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropIndexOp.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropIndexOp.java
index 23cb536d799..3c233604cbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropIndexOp.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropIndexOp.java
@@ -22,11 +22,14 @@ import org.apache.doris.analysis.AlterTableClause;
import org.apache.doris.analysis.DropIndexClause;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
+import org.apache.doris.info.PartitionNamesInfo;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.qe.ConnectContext;
import org.apache.commons.lang3.StringUtils;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
/**
@@ -38,12 +41,26 @@ public class DropIndexOp extends AlterTableOp {
private boolean alter;
+ private final PartitionNamesInfo partitionNamesInfo;
+
+ /**
+ * DropIndexOp without partition spec.
+ */
public DropIndexOp(String indexName, boolean ifExists, TableNameInfo
tableName, boolean alter) {
+ this(indexName, ifExists, tableName, alter, null);
+ }
+
+ /**
+ * DropIndexOp with optional partition spec.
+ */
+ public DropIndexOp(String indexName, boolean ifExists, TableNameInfo
tableName, boolean alter,
+ PartitionNamesInfo partitionNamesInfo) {
super(AlterOpType.SCHEMA_CHANGE);
this.indexName = indexName;
this.ifExists = ifExists;
this.tableName = tableName;
this.alter = alter;
+ this.partitionNamesInfo = partitionNamesInfo;
}
public String getIndexName() {
@@ -62,6 +79,16 @@ public class DropIndexOp extends AlterTableOp {
return alter;
}
+ public boolean hasPartitionSpec() {
+ return partitionNamesInfo != null;
+ }
+
+ public List<String> getPartitionNames() {
+ return partitionNamesInfo == null
+ ? Collections.emptyList()
+ : partitionNamesInfo.getPartitionNames();
+ }
+
@Override
public Map<String, String> getProperties() {
return null;
@@ -75,12 +102,24 @@ public class DropIndexOp extends AlterTableOp {
if (tableName != null) {
tableName.analyze(ctx);
}
+ if (partitionNamesInfo != null) {
+ if (partitionNamesInfo.isTemp()) {
+ throw new AnalysisException(
+ "DROP INDEX ON PARTITION does not support temporary
partitions");
+ }
+ if (partitionNamesInfo.getPartitionNames() == null
+ || partitionNamesInfo.getPartitionNames().isEmpty()) {
+ throw new AnalysisException(
+ "DROP INDEX ON PARTITION requires explicit partition
names, "
+ + "PARTITIONS (*) is not supported");
+ }
+ }
}
@Override
public AlterTableClause translateToLegacyAlterClause() {
return new DropIndexClause(indexName, ifExists, tableName != null ?
tableName.transferToTableName() : null,
- alter);
+ alter, partitionNamesInfo);
}
@Override
@@ -100,6 +139,9 @@ public class DropIndexOp extends AlterTableOp {
if (!alter) {
stringBuilder.append(" ON ").append(tableName != null ?
tableName.toSql() : null);
}
+ if (partitionNamesInfo != null) {
+ stringBuilder.append(" ").append(partitionNamesInfo.toSql());
+ }
return stringBuilder.toString();
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
index a0081b5b5df..be9f6a678b7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
@@ -36,12 +36,17 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.UserException;
+import org.apache.doris.info.PartitionNamesInfo;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp;
+import org.apache.doris.nereids.trees.plans.commands.info.DropIndexOp;
+import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.task.AgentTask;
import org.apache.doris.task.AgentTaskQueue;
@@ -797,4 +802,130 @@ public class IndexChangeJobTest {
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(AlterJobV2.JobState.CANCELLED,
jobV2.getJobState());
}
+
+ @Test
+ public void testDropIndexOnPartitionValidateRejectsStarPartition() throws
Exception {
+ // PARTITIONS (*) should be rejected: partitionNames is null
+ PartitionNamesInfo starPartition = new PartitionNamesInfo(true);
+ DropIndexOp dropIndexOp = new DropIndexOp("index1", false, null, true,
starPartition);
+ try {
+ dropIndexOp.validate(new ConnectContext());
+ Assert.fail("Should throw AnalysisException for PARTITIONS (*)");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("PARTITIONS (*) is not
supported"));
+ }
+ }
+
+ @Test
+ public void testDropIndexOnPartitionValidateRejectsTempPartition() throws
Exception {
+ // TEMPORARY PARTITION should be rejected
+ PartitionNamesInfo tempPartition = new PartitionNamesInfo(true,
Lists.newArrayList("p1"));
+ DropIndexOp dropIndexOp = new DropIndexOp("index1", false, null, true,
tempPartition);
+ try {
+ dropIndexOp.validate(new ConnectContext());
+ Assert.fail("Should throw AnalysisException for TEMPORARY
PARTITION");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("does not support
temporary partitions"));
+ }
+ }
+
+ @Test
+ public void testDropIndexOnPartitionValidateAcceptsNormalPartition()
throws Exception {
+ // Normal partition spec should pass validate
+ PartitionNamesInfo normalPartition = new PartitionNamesInfo(false,
Lists.newArrayList("p1", "p2"));
+ DropIndexOp dropIndexOp = new DropIndexOp("index1", false, null, true,
normalPartition);
+ // Should not throw
+ dropIndexOp.validate(new ConnectContext());
+ Assert.assertTrue(dropIndexOp.hasPartitionSpec());
+ Assert.assertEquals(2, dropIndexOp.getPartitionNames().size());
+ DropIndexClause clause = (DropIndexClause)
dropIndexOp.translateToLegacyAlterClause();
+ Assert.assertTrue(clause.hasPartitionSpec());
+ Assert.assertEquals(normalPartition.getPartitionNames(),
clause.getPartitionNames());
+ }
+
+ @Test
+ public void testDropIndexOnPartitionRejectsNonPartitionedTable() throws
UserException {
+ // testTable1 uses SinglePartitionInfo (non-partitioned), so DROP
INDEX ON PARTITION should fail
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ Database db =
masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1);
+ OlapTable olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId1);
+ String indexName = "index1";
+ TableNameInfo tableName = new
TableNameInfo(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+ // First create an inverted index
+ IndexDefinition indexDefinition = new IndexDefinition(indexName, false,
+ Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()),
+ "INVERTED",
+ Maps.newHashMap(), "balabala");
+ CreateIndexOp createIndexOp = new CreateIndexOp(tableName,
indexDefinition, false);
+ ConnectContext connectContext = new ConnectContext();
+ createIndexOp.validate(connectContext);
+ alterClauses.add(createIndexOp.translateToLegacyAlterClause());
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ alterClauses.clear();
+
+ // Now try DROP INDEX ON PARTITION on this non-partitioned table
+ PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false,
Lists.newArrayList("p1"));
+ DropIndexOp dropIndexOp = new DropIndexOp(indexName, false, tableName,
false, partitionNamesInfo);
+ dropIndexOp.validate(connectContext);
+ alterClauses.add(dropIndexOp.translateToLegacyAlterClause());
+ try {
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Assert.fail("Should throw DdlException for non-partitioned table");
+ } catch (DdlException e) {
+ Assert.assertTrue(e.getMessage().contains("is not partitioned"));
+ }
+ // Index definition should still exist
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ }
+
+ @Test
+ public void testDropIndexOnPartitionRejectsNonExistentIndex() throws
UserException {
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ Database db =
masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1);
+ OlapTable olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId1);
+ TableNameInfo tableName = new
TableNameInfo(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+
+ PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false,
Lists.newArrayList("p1"));
+ DropIndexOp dropIndexOp = new DropIndexOp("non_existent_index", false,
tableName, false, partitionNamesInfo);
+ dropIndexOp.validate(new ConnectContext());
+ alterClauses.add(dropIndexOp.translateToLegacyAlterClause());
+ try {
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Assert.fail("Should throw DdlException for non-existent index");
+ } catch (DdlException e) {
+ Assert.assertTrue(e.getMessage().contains("does not exist"));
+ }
+ }
+
+ @Test
+ public void testDropIndexOnPartitionIfExistsNonExistentIndex() throws
UserException {
+ // IF EXISTS with non-existent index and partition spec should
silently succeed
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ Database db =
masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1);
+ OlapTable olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId1);
+ TableNameInfo tableName = new
TableNameInfo(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+
+ PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false,
Lists.newArrayList("p1"));
+ DropIndexOp dropIndexOp = new DropIndexOp("non_existent_index", true,
tableName, false, partitionNamesInfo);
+ dropIndexOp.validate(new ConnectContext());
+ alterClauses.add(dropIndexOp.translateToLegacyAlterClause());
+ // IF EXISTS succeeds without changing metadata.
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
index ecf6a1be2cc..c9afc5f5d07 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
@@ -26,7 +26,10 @@ import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Table;
+import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.indexpolicy.DropIndexPolicyLog;
@@ -56,6 +59,8 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
public class SchemaChangeHandlerTest extends TestWithFeService {
private static final Logger LOG =
LogManager.getLogger(SchemaChangeHandlerTest.class);
@@ -853,6 +858,179 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
}
}
+ private OlapTable createPartitionDropAdmissionTable(String tableName)
throws Exception {
+ dropTable("test." + tableName, false);
+ createTable("CREATE TABLE test." + tableName + " ("
+ + "k1 INT, v1 STRING, INDEX idx_v1 (v1) USING INVERTED) "
+ + "DUPLICATE KEY(k1) "
+ + "PARTITION BY RANGE(k1) (PARTITION p1 VALUES LESS THAN (10),"
+ + " PARTITION p2 VALUES LESS THAN (20)) "
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1 "
+ + "PROPERTIES ('replication_num' = '1', 'light_schema_change'
= 'true');");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException("test");
+ return (OlapTable) db.getTableOrMetaException(tableName,
Table.TableType.OLAP);
+ }
+
+ @Test
+ public void testPartitionDropRequiresLightIndexChanges() throws Exception {
+ String tableName = "sc_partition_drop_disabled";
+ OlapTable tbl = createPartitionDropAdmissionTable(tableName);
+ SchemaChangeHandler handler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ int jobsBefore = handler.getIndexChangeJobs().size();
+ int schemaVersion =
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()).getSchemaVersion();
+ boolean saved = Config.enable_light_index_change;
+ try {
+ Config.enable_light_index_change = false;
+ Exception error = Assertions.assertThrows(Exception.class,
+ () -> alterTable("ALTER TABLE test." + tableName
+ + " DROP INDEX idx_v1 PARTITION (p1)",
connectContext));
+ Assertions.assertTrue(error.getMessage().contains("requires
enable_light_index_change"),
+ error.getMessage());
+ Assertions.assertEquals(jobsBefore,
handler.getIndexChangeJobs().size());
+ Assertions.assertEquals(schemaVersion,
+
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()).getSchemaVersion());
+ Assertions.assertEquals(1, tbl.getIndexes().size());
+ } finally {
+ Config.enable_light_index_change = saved;
+ }
+ }
+
+ @Test
+ public void testPartitionDropChecksAllConflictsBeforePublishingJobs()
throws Exception {
+ String tableName = "sc_partition_drop_late_conflict";
+ OlapTable tbl = createPartitionDropAdmissionTable(tableName);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException("test");
+ SchemaChangeHandler handler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ List<Partition> partitions = Lists.newArrayList(tbl.getPartitions());
+ Assertions.assertEquals(2, partitions.size());
+ Partition busyPartition = partitions.get(partitions.size() - 1);
+ long jobId = Env.getCurrentEnv().getNextId();
+ IndexChangeJob job = new IndexChangeJob(jobId, db.getId(),
tbl.getId(), tbl.getName(), 1000L, 0,
+ Lists.newArrayList(), Lists.newArrayList());
+ job.setOriginIndexId(tbl.getBaseIndexId());
+ job.setPartitionId(busyPartition.getId());
+ job.setPartitionName(busyPartition.getName());
+ job.setAlterInvertedIndexInfo(false,
Lists.newArrayList(tbl.getIndexes().get(0)));
+ handler.getIndexChangeJobs().put(jobId, job);
+ Set<Long> jobsBefore =
Sets.newHashSet(handler.getIndexChangeJobs().keySet());
+ try {
+ Assertions.assertThrows(Exception.class,
+ () -> alterTable("ALTER TABLE test." + tableName
+ + " DROP INDEX idx_v1 PARTITIONS (p1, p2)",
connectContext));
+ Assertions.assertEquals(jobsBefore,
handler.getIndexChangeJobs().keySet(),
+ "A rejected statement must not publish a deletion job for
an earlier partition");
+ Assertions.assertEquals(1, tbl.getIndexes().size());
+ } finally {
+ for (IndexChangeJob current :
Lists.newArrayList(handler.getIndexChangeJobs().values())) {
+ if (!jobsBefore.contains(current.getJobId())) {
+ current.setJobState(IndexChangeJob.JobState.CANCELLED);
+ handler.getIndexChangeJobs().remove(current.getJobId());
+ }
+ }
+ handler.getIndexChangeJobs().remove(jobId);
+ }
+ }
+
+ @Test
+ public void testDropIndexOnPartitionsMustBeTheOnlyClause() throws
Exception {
+ String tableName = "sc_drop_index_on_partitions_alone";
+ dropTable("test." + tableName, false);
+ createTable("CREATE TABLE test." + tableName + " (\n"
+ + "k1 INT,\n"
+ + "vi INT,\n"
+ + "v1 STRING,\n"
+ + "v2 STRING,\n"
+ + "INDEX idx_v1 (v1) USING INVERTED,\n"
+ + "INDEX idx_v2 (v2) USING INVERTED\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "PARTITION BY RANGE(k1) (PARTITION p1 VALUES LESS THAN
(\"10\"),"
+ + " PARTITION p2 VALUES LESS THAN (\"20\"))\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES ('replication_num' = '1', 'light_schema_change'
= 'true');");
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException("test");
+ OlapTable tbl = (OlapTable) db.getTableOrMetaException(tableName,
Table.TableType.OLAP);
+ SchemaChangeHandler handler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ int schemaVersion =
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()).getSchemaVersion();
+ int indexChangeJobCount = handler.getIndexChangeJobs().size();
+
+ // Reject mixed work before it can change columns, indexes, or
partition scopes.
+ for (String clauses : Lists.newArrayList(
+ "MODIFY COLUMN vi BIGINT NULL, DROP INDEX idx_v1 PARTITION
(p1)",
+ "ADD COLUMN x INT, DROP INDEX idx_v1 PARTITION (p1)",
+ "DROP INDEX idx_v1 PARTITION (p1), DROP INDEX idx_v2",
+ "DROP INDEX idx_v1 PARTITION (p1), DROP INDEX idx_v2 PARTITION
(p2)")) {
+ String sql = "ALTER TABLE test." + tableName + " " + clauses;
+ Exception e = Assertions.assertThrows(Exception.class, () ->
alterTable(sql, connectContext));
+ Assertions.assertTrue(e.getMessage().contains("DROP INDEX on
partitions can not be combined"),
+ e.getMessage());
+ }
+ Assertions.assertEquals(PrimitiveType.INT,
tbl.getColumn("vi").getType().getPrimitiveType());
+ Assertions.assertNull(tbl.getColumn("x"));
+ Assertions.assertEquals(Lists.newArrayList("idx_v1", "idx_v2"),
+
tbl.getIndexes().stream().map(Index::getIndexName).collect(Collectors.toList()));
+ Assertions.assertEquals(schemaVersion,
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()).getSchemaVersion());
+ Assertions.assertEquals(indexChangeJobCount,
handler.getIndexChangeJobs().size());
+
+ // Alone, it still submits an index job for the partition and keeps
the index in the table metadata.
+ alterTable("ALTER TABLE test." + tableName + " DROP INDEX idx_v1
PARTITION (p1)", connectContext);
+ Assertions.assertEquals(indexChangeJobCount + 1,
handler.getIndexChangeJobs().size());
+ Assertions.assertEquals(Lists.newArrayList("idx_v1", "idx_v2"),
+
tbl.getIndexes().stream().map(Index::getIndexName).collect(Collectors.toList()));
+ }
+
+ @Test
+ public void testDropIndexRejectedByPendingBuildOfSameIndex() throws
Exception {
+ String tableName = "sc_drop_index_pending_build";
+ dropTable("test." + tableName, false);
+ createTable("CREATE TABLE test." + tableName + " (\n"
+ + "k1 INT,\n"
+ + "v1 STRING,\n"
+ + "INDEX idx_v1 (v1) USING INVERTED\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "PARTITION BY RANGE(k1) (PARTITION p1 VALUES LESS THAN
(\"10\"),"
+ + " PARTITION p2 VALUES LESS THAN (\"20\"))\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES ('replication_num' = '1', 'light_schema_change'
= 'true');");
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException("test");
+ OlapTable tbl = (OlapTable) db.getTableOrMetaException(tableName,
Table.TableType.OLAP);
+ SchemaChangeHandler handler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ Index index = tbl.getIndexes().stream()
+ .filter(idx -> idx.getIndexName().equals("idx_v1"))
+ .findFirst().get();
+
+ // Register an unfinished build job without scheduling it.
+ // A later build must not restore a dropped index.
+ long jobId = Env.getCurrentEnv().getNextId();
+ IndexChangeJob job = new IndexChangeJob(jobId, db.getId(),
tbl.getId(), tbl.getName(), 1000L, 0,
+ Lists.newArrayList(), Lists.newArrayList());
+ job.setOriginIndexId(tbl.getBaseIndexId());
+ job.setPartitionName("p1");
+ job.setAlterInvertedIndexInfo(false, Lists.newArrayList(index));
+ handler.getIndexChangeJobs().put(jobId, job);
+ try {
+ String[][] clauseAndError = {
+ {"DROP INDEX idx_v1", "has an unfinished index change
job"},
+ {"DROP INDEX idx_v1 PARTITION (p1)", "has an unfinished
index change job"}};
+ for (String[] ce : clauseAndError) {
+ String sql = "ALTER TABLE test." + tableName + " " + ce[0];
+ Exception e = Assertions.assertThrows(Exception.class, () ->
alterTable(sql, connectContext));
+ Assertions.assertTrue(e.getMessage().contains(ce[1]),
e.getMessage());
+ }
+ Assertions.assertTrue(tbl.getIndexes().stream().anyMatch(idx ->
idx.getIndexName().equals("idx_v1")));
+
+ job.setJobState(IndexChangeJob.JobState.FINISHED);
+ alterTable("ALTER TABLE test." + tableName + " DROP INDEX idx_v1",
connectContext);
+ jobSize++;
+ Assertions.assertTrue(tbl.getIndexes().stream().noneMatch(idx ->
idx.getIndexName().equals("idx_v1")));
+ } finally {
+ handler.getIndexChangeJobs().remove(jobId);
+ }
+ }
+
@Test
public void testAddInvertedIndexStoresCanonicalBuiltinAnalyzer() throws
Exception {
createAnalyzerAliasTable("sc_ik_alias");
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy
new file mode 100644
index 00000000000..3ebdb824a59
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/index_change/test_drop_index_on_partition.groovy
@@ -0,0 +1,269 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_drop_index_on_partition", "inverted_index") {
+ def timeout = 300000
+ def delta_time = 1000
+
+ def get_build_index_job_count = { table_name ->
+ def res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";"""
+ return res.size()
+ }
+
+ def wait_for_build_index_on_partition_finish = { table_name,
expected_job_count, OpTimeout ->
+ assertTrue(expected_job_count > 0, "No index change jobs were created
for ${table_name}")
+ def finished = false
+ for (int t = delta_time; t <= OpTimeout; t += delta_time) {
+ def alter_res = sql """SHOW BUILD INDEX WHERE TableName =
"${table_name}";"""
+ assertTrue(!alter_res.any { it[7] == "CANCELLED" },
+ "An index change job was cancelled: ${alter_res}")
+ if (alter_res.size() >= expected_job_count && alter_res.every {
it[7] == "FINISHED" }) {
+ logger.info(table_name + " all index change jobs finished: " +
alter_res)
+ finished = true
+ break
+ }
+ sleep(delta_time)
+ }
+ assertTrue(finished, "Index change jobs did not finish within
${OpTimeout} ms for ${table_name}")
+ }
+
+ // case 1: basic DROP INDEX ON PARTITION with job count verification
+ def tableName1 = "test_drop_idx_on_partition_basic"
+ sql "DROP TABLE IF EXISTS ${tableName1}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ k1 int NOT NULL,
+ v1 varchar(50),
+ v2 string
+ )
+ DUPLICATE KEY(`k1`)
+ PARTITION BY RANGE(`k1`) (
+ PARTITION p1 VALUES LESS THAN (100),
+ PARTITION p2 VALUES LESS THAN (200),
+ PARTITION p3 VALUES LESS THAN (300)
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ properties("replication_num" = "1");
+ """
+
+ sql "INSERT INTO ${tableName1} VALUES (10, 'hello world', 'foo bar'), (50,
'test data', 'baz qux')"
+ sql "INSERT INTO ${tableName1} VALUES (110, 'hello doris', 'abc def'),
(150, 'test query', 'ghi jkl')"
+ sql "INSERT INTO ${tableName1} VALUES (210, 'hello index', 'mno pqr'),
(250, 'test drop', 'stu vwx')"
+
+ // create inverted index
+ sql "CREATE INDEX idx_v2 ON ${tableName1}(v2) USING INVERTED"
+ // Build physical files before testing partition deletion.
+ build_index_on_table("idx_v2", tableName1)
+ wait_for_build_index_on_partition_finish(tableName1,
get_build_index_job_count(tableName1), timeout)
+
+ // verify index exists
+ def show_idx = sql "SHOW INDEX FROM ${tableName1}"
+ logger.info("show index: " + show_idx)
+ assertTrue(show_idx.toString().contains("idx_v2"))
+
+ // record job count before DROP
+ def job_count_before = get_build_index_job_count(tableName1)
+ logger.info("job count before drop: " + job_count_before)
+
+ // drop index on partition p1 only — should create new job(s)
+ sql "DROP INDEX idx_v2 ON ${tableName1} PARTITION (p1)"
+
+ def job_count_after = get_build_index_job_count(tableName1)
+ logger.info("job count after drop: " + job_count_after)
+ assertTrue(job_count_after > job_count_before,
+ "DROP INDEX ON PARTITION should create new build index jobs, " +
+ "before: ${job_count_before}, after: ${job_count_after}")
+
+ wait_for_build_index_on_partition_finish(tableName1, job_count_after,
timeout)
+
+ // verify index definition still exists in table schema
+ show_idx = sql "SHOW INDEX FROM ${tableName1}"
+ logger.info("show index after drop on partition: " + show_idx)
+ assertTrue(show_idx.toString().contains("idx_v2"))
+
+ // query with fallback=true should return correct results on all partitions
+ sql """set enable_fallback_on_missing_inverted_index = true"""
+ def result1 = sql """SELECT * FROM ${tableName1} WHERE v2 = 'foo bar'
ORDER BY k1"""
+ assertEquals(1, result1.size())
+ assertEquals(10, result1[0][0])
+
+ def result2 = sql """SELECT * FROM ${tableName1} WHERE v2 = 'abc def'
ORDER BY k1"""
+ assertEquals(1, result2.size())
+ assertEquals(110, result2[0][0])
+
+ // case 2: drop index on multiple partitions with job count verification
+ def tableName2 = "test_drop_idx_on_multi_partitions"
+ sql "DROP TABLE IF EXISTS ${tableName2}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ k1 int NOT NULL,
+ v1 varchar(50)
+ )
+ DUPLICATE KEY(`k1`)
+ PARTITION BY RANGE(`k1`) (
+ PARTITION p1 VALUES LESS THAN (100),
+ PARTITION p2 VALUES LESS THAN (200),
+ PARTITION p3 VALUES LESS THAN (300)
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ properties("replication_num" = "1");
+ """
+
+ sql "INSERT INTO ${tableName2} VALUES (10, 'hello'), (110, 'world'), (210,
'doris')"
+
+ sql "CREATE INDEX idx_v1 ON ${tableName2}(v1) USING INVERTED"
+ build_index_on_table("idx_v1", tableName2)
+ wait_for_build_index_on_partition_finish(tableName2,
get_build_index_job_count(tableName2), timeout)
+
+ job_count_before = get_build_index_job_count(tableName2)
+
+ // drop index on p1 and p2 — should create at least 2 new jobs
+ sql "DROP INDEX idx_v1 ON ${tableName2} PARTITIONS (p1, p2)"
+
+ job_count_after = get_build_index_job_count(tableName2)
+ logger.info("multi-partition drop: job count before=${job_count_before},
after=${job_count_after}")
+ assertTrue(job_count_after >= job_count_before + 2,
+ "DROP INDEX ON 2 PARTITIONS should create at least 2 new jobs")
+
+ wait_for_build_index_on_partition_finish(tableName2, job_count_after,
timeout)
+
+ // index definition should still exist
+ show_idx = sql "SHOW INDEX FROM ${tableName2}"
+ assertTrue(show_idx.toString().contains("idx_v1"))
+
+ // queries should return correct results with fallback
+ sql """set enable_fallback_on_missing_inverted_index = true"""
+ def result3 = sql """SELECT * FROM ${tableName2} ORDER BY k1"""
+ assertEquals(3, result3.size())
+
+ // case 3: error - non-partitioned table
+ def tableName3 = "test_drop_idx_on_partition_non_partitioned"
+ sql "DROP TABLE IF EXISTS ${tableName3}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName3} (
+ k1 int NOT NULL,
+ v1 varchar(50)
+ )
+ DUPLICATE KEY(`k1`)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ properties("replication_num" = "1");
+ """
+ sql "INSERT INTO ${tableName3} VALUES (1, 'test')"
+ sql "CREATE INDEX idx_v1 ON ${tableName3}(v1) USING INVERTED"
+ build_index_on_table("idx_v1", tableName3)
+ wait_for_build_index_on_partition_finish(tableName3,
get_build_index_job_count(tableName3), timeout)
+
+ test {
+ sql "DROP INDEX idx_v1 ON ${tableName3} PARTITION (p1)"
+ exception "is not partitioned"
+ }
+
+ // case 4: error - non-inverted index type
+ def tableName4 = "test_drop_idx_on_partition_non_inverted"
+ sql "DROP TABLE IF EXISTS ${tableName4}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName4} (
+ k1 int NOT NULL,
+ v1 varchar(50)
+ )
+ DUPLICATE KEY(`k1`)
+ PARTITION BY RANGE(`k1`) (
+ PARTITION p1 VALUES LESS THAN (100),
+ PARTITION p2 VALUES LESS THAN (200)
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ properties("replication_num" = "1");
+ """
+ sql "INSERT INTO ${tableName4} VALUES (10, 'test'), (110, 'data')"
+ sql "CREATE INDEX idx_ngram ON ${tableName4}(v1) USING NGRAM_BF
PROPERTIES(\"gram_size\"=\"3\", \"bf_size\"=\"1024\")"
+ wait_for_last_col_change_finish(tableName4, timeout)
+
+ test {
+ sql "DROP INDEX idx_ngram ON ${tableName4} PARTITION (p1)"
+ exception "Only inverted index supports DROP INDEX ON PARTITION"
+ }
+
+ // case 5: error - non-existent partition
+ test {
+ sql "DROP INDEX idx_v2 ON ${tableName1} PARTITION
(non_existent_partition)"
+ exception "does not exist"
+ }
+
+ // case 6: error - non-existent index
+ test {
+ sql "DROP INDEX non_existent_idx ON ${tableName1} PARTITION (p1)"
+ exception "does not exist"
+ }
+
+ // case 7: IF EXISTS with non-existent index should succeed silently
+ sql "DROP INDEX IF EXISTS non_existent_idx ON ${tableName1} PARTITION (p1)"
+
+ // case 8: error - PARTITIONS (*) not supported
+ test {
+ sql "DROP INDEX idx_v2 ON ${tableName1} PARTITIONS (*)"
+ exception "PARTITIONS (*) is not supported"
+ }
+
+ // case 9: error - TEMPORARY PARTITION not supported
+ test {
+ sql "DROP INDEX idx_v2 ON ${tableName1} TEMPORARY PARTITION (p1)"
+ exception "does not support temporary partitions"
+ }
+
+ // case 10: ALTER TABLE form with job count verification
+ def tableName5 = "test_drop_idx_on_partition_alter"
+ sql "DROP TABLE IF EXISTS ${tableName5}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName5} (
+ k1 int NOT NULL,
+ v1 string
+ )
+ DUPLICATE KEY(`k1`)
+ PARTITION BY RANGE(`k1`) (
+ PARTITION p1 VALUES LESS THAN (100),
+ PARTITION p2 VALUES LESS THAN (200)
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ properties("replication_num" = "1");
+ """
+ sql "INSERT INTO ${tableName5} VALUES (10, 'hello world'), (110, 'test
data')"
+
+ sql "CREATE INDEX idx_v1 ON ${tableName5}(v1) USING INVERTED"
+ build_index_on_table("idx_v1", tableName5)
+ wait_for_build_index_on_partition_finish(tableName5,
get_build_index_job_count(tableName5), timeout)
+
+ job_count_before = get_build_index_job_count(tableName5)
+
+ // use ALTER TABLE form
+ sql "ALTER TABLE ${tableName5} DROP INDEX idx_v1 PARTITION (p1)"
+
+ job_count_after = get_build_index_job_count(tableName5)
+ assertTrue(job_count_after > job_count_before,
+ "ALTER TABLE DROP INDEX ON PARTITION should create new jobs")
+
+ wait_for_build_index_on_partition_finish(tableName5, job_count_after,
timeout)
+
+ // index definition should still exist
+ show_idx = sql "SHOW INDEX FROM ${tableName5}"
+ assertTrue(show_idx.toString().contains("idx_v1"))
+
+ // query all data with fallback
+ sql """set enable_fallback_on_missing_inverted_index = true"""
+ def result4 = sql """SELECT * FROM ${tableName5} ORDER BY k1"""
+ assertEquals(2, result4.size())
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
