This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch cs_opt_version-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/cs_opt_version-3.1 by this
push:
new f3048ce50ad fix cloud meta for segment v3 (#58590)
f3048ce50ad is described below
commit f3048ce50ad61e5270f6e8a0ad7a07d7e11aad40
Author: lihangyu <[email protected]>
AuthorDate: Tue Dec 2 05:05:16 2025 +0800
fix cloud meta for segment v3 (#58590)
---
.../org/apache/doris/alter/CloudRollupJobV2.java | 43 ++++++++++-----------
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 2 +-
.../apache/doris/cloud/backup/CloudRestoreJob.java | 44 +++++++++++-----------
.../cloud/common/util/CloudPropertyAnalyzer.java | 1 -
.../cloud/datasource/CloudInternalCatalog.java | 15 +++++++-
.../ext_meta/test_storage_format_v2_1.groovy | 6 +--
6 files changed, 62 insertions(+), 49 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index 4ef9ab42780..fcf7409c430 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -214,27 +214,28 @@ public class CloudRollupJobV2 extends RollupJobV2 {
for (Tablet rollupTablet : rollupIndex.getTablets()) {
OlapFile.TabletMetaCloudPB.Builder builder =
((CloudInternalCatalog)
Env.getCurrentInternalCatalog())
- .createTabletMetaBuilder(tableId, rollupIndexId,
- partitionId, rollupTablet, tabletType,
rollupSchemaHash,
- rollupKeysType, rollupShortKeyColumnCount,
tbl.getCopiedBfColumns(),
- tbl.getBfFpp(), null, rollupSchema,
- tbl.getDataSortInfo(),
tbl.getCompressionType(), tbl.getStoragePolicy(),
- tbl.isInMemory(), true,
- tbl.getName(), tbl.getTTLSeconds(),
- tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.storeRowColumn(),
- tbl.getBaseSchemaVersion(),
tbl.getCompactionPolicy(),
-
tbl.getTimeSeriesCompactionGoalSizeMbytes(),
-
tbl.getTimeSeriesCompactionFileCountThreshold(),
-
tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
-
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
-
tbl.getTimeSeriesCompactionLevelThreshold(),
- tbl.disableAutoCompaction(),
-
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
- tbl.getEnableMowLightDelete(), null,
- tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(),
tbl.getTDEAlgorithmPB(),
- tbl.storageDictPageSize(), true);
+ .createTabletMetaBuilder(tableId,
rollupIndexId,
+ partitionId, rollupTablet, tabletType,
rollupSchemaHash,
+ rollupKeysType,
rollupShortKeyColumnCount, tbl.getCopiedBfColumns(),
+ tbl.getBfFpp(), null, rollupSchema,
+ tbl.getDataSortInfo(),
tbl.getCompressionType(), tbl.getStorageFormat(),
+ tbl.getStoragePolicy(),
+ tbl.isInMemory(), true,
+ tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.storeRowColumn(),
+ tbl.getBaseSchemaVersion(),
tbl.getCompactionPolicy(),
+
tbl.getTimeSeriesCompactionGoalSizeMbytes(),
+
tbl.getTimeSeriesCompactionFileCountThreshold(),
+
tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
+
tbl.getTimeSeriesCompactionLevelThreshold(),
+ tbl.disableAutoCompaction(),
+
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
+ tbl.getEnableMowLightDelete(), null,
+ tbl.rowStorePageSize(),
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize(),
tbl.getTDEAlgorithmPB(),
+ tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 50a7f2c33da..2a2b32e693d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -258,7 +258,7 @@ public class CloudSchemaChangeJobV2 extends
SchemaChangeJobV2 {
tbl.getPartitionInfo().getTabletType(partitionId),
shadowSchemaHash, originKeysType,
shadowShortKeyColumnCount, bfColumns,
bfFpp, tabletIndexes,
shadowSchema, tbl.getDataSortInfo(),
- tbl.getCompressionType(),
+ tbl.getCompressionType(),
tbl.getStorageFormat(),
tbl.getStoragePolicy(),
tbl.isInMemory(), true,
tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index 4f0a7bce1b9..1a1bfb296ee 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -375,27 +375,29 @@ public class CloudRestoreJob extends RestoreJob {
try {
requestBuilder.addTabletMetas(((CloudInternalCatalog)
Env.getCurrentInternalCatalog())
.createTabletMetaBuilder(localTbl.getId(),
restoredIdx.getId(),
- restorePart.getId(), restoreTablet,
-
localTbl.getPartitionInfo().getTabletType(restorePart.getId()),
- indexMeta.getSchemaHash(),
indexMeta.getKeysType(),
- indexMeta.getShortKeyColumnCount(),
localTbl.getCopiedBfColumns(),
- localTbl.getBfFpp(), indexes,
indexMeta.getSchema(), localTbl.getDataSortInfo(),
- localTbl.getCompressionType(),
localTbl.getStoragePolicy(),
- localTbl.isInMemory(), false,
localTbl.getName(), localTbl.getTTLSeconds(),
- localTbl.getEnableUniqueKeyMergeOnWrite(),
localTbl.storeRowColumn(),
- localTbl.getBaseSchemaVersion(),
localTbl.getCompactionPolicy(),
-
localTbl.getTimeSeriesCompactionGoalSizeMbytes(),
-
localTbl.getTimeSeriesCompactionFileCountThreshold(),
-
localTbl.getTimeSeriesCompactionTimeThresholdSeconds(),
-
localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
-
localTbl.getTimeSeriesCompactionLevelThreshold(),
localTbl.disableAutoCompaction(),
-
localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
- localTbl.getEnableMowLightDelete(),
-
localTbl.getInvertedIndexFileStorageFormat(),
- localTbl.rowStorePageSize(),
- localTbl.variantEnableFlattenNested(),
- localTbl.storagePageSize(),
localTbl.getTDEAlgorithmPB(),
- localTbl.storageDictPageSize(), false));
+ restorePart.getId(), restoreTablet,
+
localTbl.getPartitionInfo().getTabletType(restorePart.getId()),
+ indexMeta.getSchemaHash(),
indexMeta.getKeysType(),
+ indexMeta.getShortKeyColumnCount(),
localTbl.getCopiedBfColumns(),
+ localTbl.getBfFpp(), indexes,
indexMeta.getSchema(), localTbl.getDataSortInfo(),
+ localTbl.getCompressionType(),
localTbl.getStorageFormat(),
+ localTbl.getStoragePolicy(),
+ localTbl.isInMemory(), false,
localTbl.getName(), localTbl.getTTLSeconds(),
+
localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(),
+ localTbl.getBaseSchemaVersion(),
localTbl.getCompactionPolicy(),
+
localTbl.getTimeSeriesCompactionGoalSizeMbytes(),
+
localTbl.getTimeSeriesCompactionFileCountThreshold(),
+
localTbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+
localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
+
localTbl.getTimeSeriesCompactionLevelThreshold(),
+ localTbl.disableAutoCompaction(),
+
localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
+ localTbl.getEnableMowLightDelete(),
+
localTbl.getInvertedIndexFileStorageFormat(),
+ localTbl.rowStorePageSize(),
+ localTbl.variantEnableFlattenNested(),
+ localTbl.storagePageSize(),
localTbl.getTDEAlgorithmPB(),
+ localTbl.storageDictPageSize(),
false));
// In cloud mode all storage medium will be saved to
HDD.
TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
restoredIdx.getId(),
indexMeta.getSchemaHash(), TStorageMedium.HDD);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
index b41570a25fc..7ac160f4b18 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
@@ -30,7 +30,6 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer {
forceProperties = ImmutableList.of(
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_INMEMORY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM),
-
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 9d516880818..457a5ca1f13 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -65,6 +65,7 @@ import org.apache.doris.rpc.RpcException;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TSortType;
+import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Preconditions;
@@ -165,7 +166,8 @@ public class CloudInternalCatalog extends InternalCatalog {
OlapFile.TabletMetaCloudPB.Builder builder =
createTabletMetaBuilder(tbl.getId(), indexId,
partitionId, tablet, tabletType, schemaHash, keysType,
shortKeyColumnCount,
bfColumns, tbl.getBfFpp(), indexes, columns,
tbl.getDataSortInfo(),
- tbl.getCompressionType(), storagePolicy, isInMemory,
false, tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getCompressionType(), tbl.getStorageFormat(),
storagePolicy, isInMemory, false,
+ tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.storeRowColumn(), indexMeta.getSchemaVersion(),
tbl.getCompactionPolicy(),
tbl.getTimeSeriesCompactionGoalSizeMbytes(),
tbl.getTimeSeriesCompactionFileCountThreshold(),
@@ -203,7 +205,7 @@ public class CloudInternalCatalog extends InternalCatalog {
long partitionId, Tablet tablet, TTabletType tabletType, int
schemaHash, KeysType keysType,
short shortKeyColumnCount, Set<String> bfColumns, double bfFpp,
List<Index> indexes,
List<Column> schemaColumns, DataSortInfo dataSortInfo,
TCompressionType compressionType,
- String storagePolicy, boolean isInMemory, boolean isShadow,
+ TStorageFormat storageFormat, String storagePolicy, boolean
isInMemory, boolean isShadow,
String tableName, long ttlSeconds, boolean
enableUniqueKeyMergeOnWrite,
boolean storeRowColumn, int schemaVersion, String compactionPolicy,
Long timeSeriesCompactionGoalSizeMbytes, Long
timeSeriesCompactionFileCountThreshold,
@@ -313,6 +315,15 @@ public class CloudInternalCatalog extends InternalCatalog {
break;
}
+ // Enable external column meta layout when storage_format is V3 (Cloud
mode).
+ switch (storageFormat) {
+ case V3:
+ schemaBuilder.setIsExternalSegmentMetaUsedDefault(true);
+ break;
+ default:
+ break;
+ }
+
schemaBuilder.setSortColNum(dataSortInfo.getColNum());
for (int i = 0; i < schemaColumns.size(); i++) {
Column column = schemaColumns.get(i);
diff --git
a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
index d7fe1e48e3a..ece39caed92 100644
--- a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
+++ b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
@@ -19,9 +19,9 @@ suite("test_storage_format_v2_1") {
def tableName = "test_storage_format_v2_1_table"
// Test 1: Create table with storage_format = V2.1
- sql "DROP TABLE IF EXISTS ${tableName}"
+ sql "DROP TABLE IF EXISTS test_storage_format_v2_1_table"
sql """
- CREATE TABLE ${tableName} (
+ CREATE TABLE test_storage_format_v2_1_table (
k bigint,
v variant
)
@@ -47,7 +47,7 @@ suite("test_storage_format_v2_1") {
// Verify table properties
def result = sql "SHOW CREATE TABLE ${tableName}"
logger.info("Show create table result: ${result}")
- assertTrue(result[0][1].contains("V3"),
+ assertTrue(result[0][1].contains("\"storage_format\" = \"V3\""),
"Table should be created with storage_format V3")
// Cleanup
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
