This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e7b931b3cbf [refactor](inverted index) Refactor the idx storage format
to avoid cyclic references in Thrift files. (#36757)
e7b931b3cbf is described below
commit e7b931b3cbfc55f566dc10cfb62d6ec6835edde2
Author: Sun Chenyang <[email protected]>
AuthorDate: Wed Jun 26 17:25:17 2024 +0800
[refactor](inverted index) Refactor the idx storage format to avoid cyclic
references in Thrift files. (#36757)
## Proposed changes
`TInvertedIndexStorageFormat` is defined in the `AgentService.thrift`.
When other Thirft files use `TInvertedIndexStorageFormat`, need to
`include AgentService.thrift`.
However, `AgentService.thrift` might already include that file, causing
cyclic references.
I have defined `TInvetedIndexFileStorageFormat` in `Types.thrift` to
represent the inverted index file format. `TInvertedIndexStorageFormat`
will be deprecated.
This will not impact the upgrade.
---
be/src/olap/tablet_meta.cpp | 28 +++++++++++++++++-----
be/src/olap/tablet_meta.h | 4 ++--
.../apache/doris/alter/SchemaChangeHandler.java | 8 -------
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 ++-
.../java/org/apache/doris/backup/RestoreJob.java | 2 +-
.../main/java/org/apache/doris/catalog/Env.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 14 +++++------
.../org/apache/doris/catalog/TableProperty.java | 16 ++++++-------
.../apache/doris/common/util/PropertyAnalyzer.java | 28 +++++++++++-----------
.../apache/doris/datasource/InternalCatalog.java | 10 ++++----
.../org/apache/doris/master/ReportHandler.java | 4 ++--
.../java/org/apache/doris/qe/ShowExecutor.java | 6 ++---
.../org/apache/doris/task/CreateReplicaTask.java | 22 +++++++++++++----
gensrc/thrift/AgentService.thrift | 4 +++-
gensrc/thrift/Types.thrift | 9 +++++++
15 files changed, 96 insertions(+), 64 deletions(-)
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 26ed8d3ee57..cced41a86ee 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -62,6 +62,22 @@ TabletMetaSharedPtr TabletMeta::create(
if (request.__isset.binlog_config) {
binlog_config = request.binlog_config;
}
+ TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format =
+ request.inverted_index_file_storage_format;
+
+ // We will discard this format. Don't make any further changes here.
+ if (request.__isset.inverted_index_storage_format) {
+ switch (request.inverted_index_storage_format) {
+ case TInvertedIndexStorageFormat::V1:
+ inverted_index_file_storage_format =
TInvertedIndexFileStorageFormat::V1;
+ break;
+ case TInvertedIndexStorageFormat::V2:
+ inverted_index_file_storage_format =
TInvertedIndexFileStorageFormat::V2;
+ break;
+ default:
+ break;
+ }
+ }
return std::make_shared<TabletMeta>(
request.table_id, request.partition_id, request.tablet_id,
request.replica_id,
request.tablet_schema.schema_hash, shard_id,
request.tablet_schema, next_unique_id,
@@ -76,7 +92,7 @@ TabletMetaSharedPtr TabletMeta::create(
request.time_series_compaction_file_count_threshold,
request.time_series_compaction_time_threshold_seconds,
request.time_series_compaction_empty_rowsets_threshold,
- request.time_series_compaction_level_threshold,
request.inverted_index_storage_format);
+ request.time_series_compaction_level_threshold,
inverted_index_file_storage_format);
}
TabletMeta::TabletMeta()
@@ -97,7 +113,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
int64_t time_series_compaction_time_threshold_seconds,
int64_t time_series_compaction_empty_rowsets_threshold,
int64_t time_series_compaction_level_threshold,
- TInvertedIndexStorageFormat::type
inverted_index_storage_format)
+ TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format)
: _tablet_uid(0, 0),
_schema(new TabletSchema),
_delete_bitmap(new DeleteBitmap(tablet_id)) {
@@ -175,15 +191,15 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
break;
}
- switch (inverted_index_storage_format) {
- case TInvertedIndexStorageFormat::V1:
+ switch (inverted_index_file_storage_format) {
+ case TInvertedIndexFileStorageFormat::V1:
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
break;
- case TInvertedIndexStorageFormat::V2:
+ case TInvertedIndexFileStorageFormat::V2:
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
break;
default:
-
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+
schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
break;
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index b13bf2f0853..32c6fde568c 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -112,8 +112,8 @@ public:
int64_t time_series_compaction_time_threshold_seconds = 3600,
int64_t time_series_compaction_empty_rowsets_threshold = 5,
int64_t time_series_compaction_level_threshold = 1,
- TInvertedIndexStorageFormat::type inverted_index_storage_format
=
- TInvertedIndexStorageFormat::V1);
+ TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format =
+ TInvertedIndexFileStorageFormat::V2);
// If need add a filed in TableMeta, filed init copy in copy construct
function
TabletMeta(const TabletMeta& tablet_meta);
TabletMeta(TabletMeta&& tablet_meta) = delete;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 8b8d9c1ac72..15f8c88bb64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -98,7 +98,6 @@ import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.ClearAlterTask;
import org.apache.doris.task.UpdateTabletMetaInfoTask;
-import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTaskType;
@@ -1344,9 +1343,6 @@ public class SchemaChangeHandler extends AlterHandler {
TStorageFormat storageFormat =
PropertyAnalyzer.analyzeStorageFormat(propertyMap);
- TInvertedIndexStorageFormat invertedIndexStorageFormat =
-
PropertyAnalyzer.analyzeInvertedIndexStorageFormat(propertyMap);
-
// property store_row_column && row_store_columns
// eg. "store_row_column" = "true"
// eg. "row_store_columns" = "k1, k2"
@@ -1436,10 +1432,6 @@ public class SchemaChangeHandler extends AlterHandler {
if (olapTable.getStorageFormat() != TStorageFormat.V2) {
needAlter = true;
}
- } else if (invertedIndexStorageFormat ==
TInvertedIndexStorageFormat.V2) {
- if (olapTable.getInvertedIndexStorageFormat() !=
TInvertedIndexStorageFormat.V2) {
- needAlter = true;
- }
}
if (!needAlter) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 7229966cc80..a0c7058f535 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -300,7 +300,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
}
-
createReplicaTask.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat());
+
createReplicaTask.setInvertedIndexFileStorageFormat(tbl
+
.getInvertedIndexFileStorageFormat());
batchTask.addTask(createReplicaTask);
} // end for rollupReplicas
} // end for rollupTablets
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 9b870af8b9d..c78b5b5202a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1127,7 +1127,7 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
binlogConfig,
localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool);
-
task.setInvertedIndexStorageFormat(localTbl.getInvertedIndexStorageFormat());
+
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
batchTask.addTask(task);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 8eff5e2bebc..207a16a2678 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3448,7 +3448,7 @@ public class Env {
// inverted index storage type
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT).append("\"
= \"");
- sb.append(olapTable.getInvertedIndexStorageFormat()).append("\"");
+ sb.append(olapTable.getInvertedIndexFileStorageFormat()).append("\"");
// compression type
if (olapTable.getCompressionType() != TCompressionType.LZ4F) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 63427b8b579..490e8a7b3a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -72,7 +72,7 @@ import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TFetchOption;
-import org.apache.doris.thrift.TInvertedIndexStorageFormat;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TOlapTable;
import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TSortType;
@@ -2493,11 +2493,11 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
tableProperty.buildStorageFormat();
}
- public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat
invertedIndexStorageFormat) {
+ public void
setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat
invertedIndexFileStorageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
- invertedIndexStorageFormat.name());
- tableProperty.buildInvertedIndexStorageFormat();
+ invertedIndexFileStorageFormat.name());
+ tableProperty.buildInvertedIndexFileStorageFormat();
}
public TStorageFormat getStorageFormat() {
@@ -2507,11 +2507,11 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return tableProperty.getStorageFormat();
}
- public TInvertedIndexStorageFormat getInvertedIndexStorageFormat() {
+ public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat()
{
if (tableProperty == null) {
- return TInvertedIndexStorageFormat.V2;
+ return TInvertedIndexFileStorageFormat.V2;
}
- return tableProperty.getInvertedIndexStorageFormat();
+ return tableProperty.getInvertedIndexFileStorageFormat();
}
public TCompressionType getCompressionType() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 51bb7f69792..59c483020fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -27,7 +27,7 @@ import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TCompressionType;
-import org.apache.doris.thrift.TInvertedIndexStorageFormat;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@@ -88,7 +88,7 @@ public class TableProperty implements Writable,
GsonPostProcessable {
*/
private TStorageFormat storageFormat = TStorageFormat.DEFAULT;
- private TInvertedIndexStorageFormat invertedIndexStorageFormat =
TInvertedIndexStorageFormat.DEFAULT;
+ private TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat =
TInvertedIndexFileStorageFormat.DEFAULT;
private TCompressionType compressionType = TCompressionType.LZ4F;
@@ -470,10 +470,10 @@ public class TableProperty implements Writable,
GsonPostProcessable {
return this;
}
- public TableProperty buildInvertedIndexStorageFormat() {
- invertedIndexStorageFormat =
TInvertedIndexStorageFormat.valueOf(properties.getOrDefault(
+ public TableProperty buildInvertedIndexFileStorageFormat() {
+ invertedIndexFileStorageFormat =
TInvertedIndexFileStorageFormat.valueOf(properties.getOrDefault(
PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
- TInvertedIndexStorageFormat.DEFAULT.name()));
+ TInvertedIndexFileStorageFormat.DEFAULT.name()));
return this;
}
@@ -540,8 +540,8 @@ public class TableProperty implements Writable,
GsonPostProcessable {
return storageFormat;
}
- public TInvertedIndexStorageFormat getInvertedIndexStorageFormat() {
- return invertedIndexStorageFormat;
+ public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat()
{
+ return invertedIndexFileStorageFormat;
}
public DataSortInfo getDataSortInfo() {
@@ -648,7 +648,7 @@ public class TableProperty implements Writable,
GsonPostProcessable {
buildMinLoadReplicaNum();
buildStorageMedium();
buildStorageFormat();
- buildInvertedIndexStorageFormat();
+ buildInvertedIndexFileStorageFormat();
buildDataSortInfo();
buildCompressionType();
buildStoragePolicy();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 52b55b193d8..29ed40676b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -42,7 +42,7 @@ import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TCompressionType;
-import org.apache.doris.thrift.TInvertedIndexStorageFormat;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TSortType;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@@ -1037,32 +1037,32 @@ public class PropertyAnalyzer {
}
}
- public static TInvertedIndexStorageFormat
analyzeInvertedIndexStorageFormat(Map<String, String> properties)
+ public static TInvertedIndexFileStorageFormat
analyzeInvertedIndexFileStorageFormat(Map<String, String> properties)
throws AnalysisException {
- String invertedIndexStorageFormat = "";
+ String invertedIndexFileStorageFormat = "";
if (properties != null &&
properties.containsKey(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT)) {
- invertedIndexStorageFormat =
properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
+ invertedIndexFileStorageFormat =
properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
} else {
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
- return TInvertedIndexStorageFormat.V1;
+ return TInvertedIndexFileStorageFormat.V1;
} else {
- return TInvertedIndexStorageFormat.V2;
+ return TInvertedIndexFileStorageFormat.V2;
}
}
- if (invertedIndexStorageFormat.equalsIgnoreCase("v1")) {
- return TInvertedIndexStorageFormat.V1;
- } else if (invertedIndexStorageFormat.equalsIgnoreCase("v2")) {
- return TInvertedIndexStorageFormat.V2;
- } else if (invertedIndexStorageFormat.equalsIgnoreCase("default")) {
+ if (invertedIndexFileStorageFormat.equalsIgnoreCase("v1")) {
+ return TInvertedIndexFileStorageFormat.V1;
+ } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v2")) {
+ return TInvertedIndexFileStorageFormat.V2;
+ } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default"))
{
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
- return TInvertedIndexStorageFormat.V1;
+ return TInvertedIndexFileStorageFormat.V1;
} else {
- return TInvertedIndexStorageFormat.V2;
+ return TInvertedIndexFileStorageFormat.V2;
}
} else {
- throw new AnalysisException("unknown inverted index storage
format: " + invertedIndexStorageFormat);
+ throw new AnalysisException("unknown inverted index storage
format: " + invertedIndexFileStorageFormat);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 25d17ad0454..f3642700c84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -166,7 +166,7 @@ import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.CreateReplicaTask;
import org.apache.doris.task.DropReplicaTask;
import org.apache.doris.thrift.TCompressionType;
-import org.apache.doris.thrift.TInvertedIndexStorageFormat;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@@ -2055,7 +2055,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
objectPool);
task.setStorageFormat(tbl.getStorageFormat());
-
task.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat());
+
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
task.setClusterKeyIndexes(clusterKeyIndexes);
batchTask.addTask(task);
// add to AgentTaskQueue for handling finish report.
@@ -2410,13 +2410,13 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
olapTable.setStorageFormat(storageFormat);
- TInvertedIndexStorageFormat invertedIndexStorageFormat;
+ TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat;
try {
- invertedIndexStorageFormat =
PropertyAnalyzer.analyzeInvertedIndexStorageFormat(properties);
+ invertedIndexFileStorageFormat =
PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(properties);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
- olapTable.setInvertedIndexStorageFormat(invertedIndexStorageFormat);
+
olapTable.setInvertedIndexFileStorageFormat(invertedIndexFileStorageFormat);
// get compression type
TCompressionType compressionType = TCompressionType.LZ4;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index fa8727a63b9..d36144af46c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -889,8 +889,8 @@ public class ReportHandler extends Daemon {
objectPool);
createReplicaTask.setIsRecoverTask(true);
-
createReplicaTask.setInvertedIndexStorageFormat(olapTable
-
.getInvertedIndexStorageFormat());
+
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
+
.getInvertedIndexFileStorageFormat());
createReplicaBatchTask.addTask(createReplicaTask);
} else {
// just set this replica as bad
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 1af7580b8e2..62e8e6dad5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -970,13 +970,13 @@ public class ShowExecutor {
}
if (showTableStmt.isVerbose()) {
String storageFormat = "NONE";
- String invertedIndexStorageFormat = "NONE";
+ String invertedIndexFileStorageFormat = "NONE";
if (tbl instanceof OlapTable) {
storageFormat = ((OlapTable)
tbl).getStorageFormat().toString();
- invertedIndexStorageFormat = ((OlapTable)
tbl).getInvertedIndexStorageFormat().toString();
+ invertedIndexFileStorageFormat = ((OlapTable)
tbl).getInvertedIndexFileStorageFormat().toString();
}
rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(),
storageFormat,
- invertedIndexStorageFormat));
+ invertedIndexFileStorageFormat));
} else {
rows.add(Lists.newArrayList(tbl.getName()));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 8a658de62bb..022a8be67a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -31,6 +31,7 @@ import org.apache.doris.policy.PolicyTypeEnum;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TCreateTabletReq;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TOlapTableIndex;
import org.apache.doris.thrift.TStatusCode;
@@ -91,7 +92,7 @@ public class CreateReplicaTask extends AgentTask {
// TODO should unify the naming of v1(alpha rowset), v2(beta rowset), it
is very confused to read code
private TStorageFormat storageFormat = TStorageFormat.V2;
- private TInvertedIndexStorageFormat invertedIndexStorageFormat =
TInvertedIndexStorageFormat.V1;
+ private TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat =
TInvertedIndexFileStorageFormat.V2;
// true if this task is created by recover request(See comment of
Config.recover_with_empty_tablet)
private boolean isRecoverTask = false;
@@ -244,8 +245,8 @@ public class CreateReplicaTask extends AgentTask {
this.storageFormat = storageFormat;
}
- public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat
invertedIndexStorageFormat) {
- this.invertedIndexStorageFormat = invertedIndexStorageFormat;
+ public void
setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat
invertedIndexFileStorageFormat) {
+ this.invertedIndexFileStorageFormat = invertedIndexFileStorageFormat;
}
public void setClusterKeyIndexes(List<Integer> clusterKeyIndexes) {
@@ -362,8 +363,19 @@ public class CreateReplicaTask extends AgentTask {
createTabletReq.setStorageFormat(storageFormat);
}
- if (invertedIndexStorageFormat != null) {
-
createTabletReq.setInvertedIndexStorageFormat(invertedIndexStorageFormat);
+ if (invertedIndexFileStorageFormat != null) {
+
createTabletReq.setInvertedIndexFileStorageFormat(invertedIndexFileStorageFormat);
+ // We will discard the "TInvertedIndexStorageFormat". Don't make
any further changes here.
+ switch (invertedIndexFileStorageFormat) {
+ case V1:
+
createTabletReq.setInvertedIndexStorageFormat(TInvertedIndexStorageFormat.V1);
+ break;
+ case V2:
+
createTabletReq.setInvertedIndexStorageFormat(TInvertedIndexStorageFormat.V2);
+ break;
+ default:
+ break;
+ }
}
createTabletReq.setTabletType(tabletType);
createTabletReq.setCompressionType(compressionType);
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 7d21fb7939e..a03cb9df99b 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -180,7 +180,9 @@ struct TCreateTabletReq {
25: optional i64 time_series_compaction_time_threshold_seconds = 3600
26: optional i64 time_series_compaction_empty_rowsets_threshold = 5
27: optional i64 time_series_compaction_level_threshold = 1
- 28: optional TInvertedIndexStorageFormat inverted_index_storage_format =
TInvertedIndexStorageFormat.V1
+ 28: optional TInvertedIndexStorageFormat inverted_index_storage_format =
TInvertedIndexStorageFormat.DEFAULT // Deprecated
+ 29: optional Types.TInvertedIndexFileStorageFormat
inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2
+
// For cloud
1000: optional bool is_in_memory = false
1001: optional bool is_persistent = false
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index f13951462a2..a13e5c03ce2 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -118,6 +118,15 @@ enum TStorageBackendType {
AZURE
}
+// Enumerates the storage formats for inverted indexes in src_backends.
+// This enum is used to distinguish between different organizational methods
+// of inverted index data, affecting how the index is stored and accessed.
+enum TInvertedIndexFileStorageFormat {
+ DEFAULT, // Default format, unspecified storage method.
+ V1, // Index per idx: Each index is stored separately based on its
identifier.
+ V2 // Segment id per idx: Indexes are organized based on segment
identifiers, grouping indexes by their associated segment.
+}
+
struct TScalarType {
1: required TPrimitiveType type
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
