This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 93518940efd branch-3.1: [Exec](meta) Support set propertiy
dict_page_size by create table #52243 (#53982)
93518940efd is described below
commit 93518940efd4333cba5169c74483e83480a47ffe
Author: HappenLee <[email protected]>
AuthorDate: Tue Jul 29 17:17:32 2025 +0800
branch-3.1: [Exec](meta) Support set propertiy dict_page_size by create
table #52243 (#53982)
cherry pick #52243
Now we can set dict page size in create tableļ¼
```
CREATE TABLE `customer_bak` (
`c_custkey` int NOT NULL,
`c_name` varchar(26) NOT NULL,
`c_address` varchar(41) NOT NULL,
`c_city` varchar(11) NOT NULL,
`c_nation` varchar(16) NOT NULL,
`c_region` varchar(13) NOT NULL,
`c_phone` varchar(16) NOT NULL,
`c_mktsegment` varchar(11) NOT NULL
) ENGINE = OLAP DUPLICATE KEY(`c_custkey`) COMMENT 'OLAP' DISTRIBUTED BY
HASH(`c_custkey`) BUCKETS 12 PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"storage_dict_page_size" = "148576",
"storage_page_size" = "148576", "min_load_replica_num" = "-1",
"is_being_synced" = "false", "colocate_with" = "groupa2",
"storage_medium" = "hdd", "storage_format" = "V2",
"inverted_index_storage_format" = "V2",
"light_schema_change" = "true", "disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);
```
some case if dict page size too bigger cause too mush memory, some case
if dict page too small, the query performance not good. so we need set
the dict page size by user self.
---
be/src/cloud/pb_convert.cpp | 4 ++
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 6 ++-
be/src/olap/rowset/segment_v2/column_writer.cpp | 1 +
be/src/olap/rowset/segment_v2/column_writer.h | 5 ++-
be/src/olap/rowset/segment_v2/options.h | 6 +--
be/src/olap/rowset/segment_v2/segment_writer.cpp | 1 +
.../rowset/segment_v2/vertical_segment_writer.cpp | 1 +
be/src/olap/tablet_meta.cpp | 3 ++
be/src/olap/tablet_schema.cpp | 4 ++
be/src/olap/tablet_schema.h | 5 +++
.../org/apache/doris/alter/CloudRollupJobV2.java | 3 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../analysis/ModifyTablePropertiesClause.java | 5 ++-
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 6 +++
.../java/org/apache/doris/catalog/OlapTable.java | 14 ++++++
.../org/apache/doris/catalog/TableProperty.java | 14 ++++++
.../apache/doris/cloud/backup/CloudRestoreJob.java | 3 +-
.../cloud/datasource/CloudInternalCatalog.java | 7 ++-
.../apache/doris/common/util/PropertyAnalyzer.java | 21 +++++++++
.../apache/doris/datasource/InternalCatalog.java | 15 ++++++-
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../org/apache/doris/task/CreateReplicaTask.java | 6 ++-
.../apache/doris/common/PropertyAnalyzerTest.java | 48 +++++++++++++++++++++
.../java/org/apache/doris/task/AgentTaskTest.java | 5 ++-
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 1 +
.../data/query_p0/system/test_table_properties.out | Bin 16279 -> 16747 bytes
30 files changed, 180 insertions(+), 21 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 202cc26bc26..491e81ba724 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -347,6 +347,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
const TabletSchemaPB
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
out->set_storage_page_size(in.storage_page_size());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
}
void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&&
in) {
@@ -376,6 +377,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
TabletSchemaPB&& in)
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
out->set_storage_page_size(in.storage_page_size());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
}
TabletSchemaPB cloud_tablet_schema_to_doris(const TabletSchemaCloudPB& in) {
@@ -418,6 +420,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
const TabletSchemaCloudPB
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
out->set_storage_page_size(in.storage_page_size());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
}
void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&&
in) {
@@ -448,6 +451,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
TabletSchemaCloudPB&& in)
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
out->set_storage_page_size(in.storage_page_size());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
}
TabletMetaCloudPB doris_tablet_meta_to_cloud(const TabletMetaPB& in) {
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index d7b8cb0d387..71cd625031c 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -55,8 +55,10 @@ Status BinaryDictPageBuilder::init() {
&data_page_builder_ptr, _options));
_data_page_builder.reset(data_page_builder_ptr);
PageBuilderOptions dict_builder_options;
- dict_builder_options.data_page_size =
- std::min(_options.data_page_size, _options.dict_page_size);
+ // here the binary plain page is used to store the dictionary items so
+ // the data page size is set to the same as the dict page size
+ dict_builder_options.data_page_size = _options.dict_page_size;
+ dict_builder_options.dict_page_size = _options.dict_page_size;
dict_builder_options.is_dict_page = true;
PageBuilder* dict_builder_ptr = nullptr;
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp
b/be/src/olap/rowset/segment_v2/column_writer.cpp
index ffc7385c03e..306bb57babd 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -446,6 +446,7 @@ Status ScalarColumnWriter::init() {
// create page builder
PageBuilderOptions opts;
opts.data_page_size = _opts.data_page_size;
+ opts.dict_page_size = _opts.dict_page_size;
RETURN_IF_ERROR(_encoding_info->create_page_builder(opts, &page_builder));
if (page_builder == nullptr) {
return Status::NotSupported("Failed to create page builder for type {}
and encoding {}",
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h
b/be/src/olap/rowset/segment_v2/column_writer.h
index 8e7398f0418..a447d95e234 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -55,7 +55,8 @@ struct ColumnWriterOptions {
// - input:
column_id/unique_id/type/length/encoding/compression/is_nullable members
// - output: encoding/indexes/dict_page members
ColumnMetaPB* meta = nullptr;
- size_t data_page_size = 64 * 1024;
+ size_t data_page_size = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ size_t dict_page_size = STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
// store compressed page only when space saving is above the threshold.
// space saving = 1 - compressed_size / uncompressed_size
double compression_min_space_saving = 0.1;
@@ -79,7 +80,7 @@ struct ColumnWriterOptions {
std::string to_string() const {
std::stringstream ss;
ss << std::boolalpha << "meta=" << meta->DebugString()
- << ", data_page_size=" << data_page_size
+ << ", data_page_size=" << data_page_size << ", dict_page_size=" <<
dict_page_size
<< ", compression_min_space_saving = " <<
compression_min_space_saving
<< ", need_zone_map=" << need_zone_map << ", need_bitmap_index=" <<
need_bitmap_index
<< ", need_bloom_filter" << need_bloom_filter;
diff --git a/be/src/olap/rowset/segment_v2/options.h
b/be/src/olap/rowset/segment_v2/options.h
index 164a8a27fc1..ad8c4d90448 100644
--- a/be/src/olap/rowset/segment_v2/options.h
+++ b/be/src/olap/rowset/segment_v2/options.h
@@ -22,15 +22,15 @@
namespace doris {
namespace segment_v2 {
-static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M
static constexpr size_t STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536;
+static constexpr size_t STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE = 256 * 1024l;
constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store
page size: 16KB
struct PageBuilderOptions {
- size_t data_page_size = DEFAULT_PAGE_SIZE;
+ size_t data_page_size = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
- size_t dict_page_size = DEFAULT_PAGE_SIZE;
+ size_t dict_page_size = STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
bool need_check_bitmap = true;
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 21cc7d3da05..a923408669f 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -261,6 +261,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid,
const TabletColumn& co
if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
opts.data_page_size = storage_page_size;
}
+ opts.dict_page_size = _tablet_schema->storage_dict_page_size();
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int64_t>(
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 444a5af15c3..276fbc95394 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -260,6 +260,7 @@ Status
VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
opts.data_page_size = storage_page_size;
}
+ opts.dict_page_size = _tablet_schema->storage_dict_page_size();
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int64_t>(
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 1160706f4a1..f0806acedc4 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -347,6 +347,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
if (tablet_schema.__isset.storage_page_size) {
schema->set_storage_page_size(tablet_schema.storage_page_size);
}
+ if (tablet_schema.__isset.storage_dict_page_size) {
+
schema->set_storage_dict_page_size(tablet_schema.storage_dict_page_size);
+ }
if (tablet_schema.__isset.skip_write_index_on_load) {
schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index b1dc2cc35bf..e20c0bbda02 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1130,6 +1130,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_compression_type = schema.compression_type();
_row_store_page_size = schema.row_store_page_size();
_storage_page_size = schema.storage_page_size();
+ _storage_dict_page_size = schema.storage_dict_page_size();
_schema_version = schema.schema_version();
// Default to V1 inverted index storage format for backward compatibility
if not specified in schema.
if (!schema.has_inverted_index_storage_format()) {
@@ -1212,6 +1213,7 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
_row_store_page_size = ori_tablet_schema.row_store_page_size();
_storage_page_size = ori_tablet_schema.storage_page_size();
_variant_enable_flatten_nested =
ori_tablet_schema.variant_flatten_nested();
+ _storage_dict_page_size = ori_tablet_schema.storage_dict_page_size();
// copy from table_schema_param
_schema_version = version;
@@ -1389,6 +1391,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
tablet_schema_pb->set_compression_type(_compression_type);
tablet_schema_pb->set_row_store_page_size(_row_store_page_size);
tablet_schema_pb->set_storage_page_size(_storage_page_size);
+ tablet_schema_pb->set_storage_dict_page_size(_storage_dict_page_size);
tablet_schema_pb->set_version_col_idx(_version_col_idx);
tablet_schema_pb->set_skip_bitmap_col_idx(_skip_bitmap_col_idx);
tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format);
@@ -1721,6 +1724,7 @@ bool operator==(const TabletSchema& a, const
TabletSchema& b) {
if (a._store_row_column != b._store_row_column) return false;
if (a._row_store_page_size != b._row_store_page_size) return false;
if (a._storage_page_size != b._storage_page_size) return false;
+ if (a._storage_dict_page_size != b._storage_dict_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return
false;
if (a._variant_enable_flatten_nested != b._variant_enable_flatten_nested)
return false;
return true;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 0343c7abf53..afcbaa0627f 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -470,6 +470,10 @@ public:
}
return 0;
}
+ void set_storage_dict_page_size(long storage_dict_page_size) {
+ _storage_dict_page_size = storage_dict_page_size;
+ }
+ long storage_dict_page_size() const { return _storage_dict_page_size; }
const std::vector<const TabletIndex*> inverted_indexes() const {
std::vector<const TabletIndex*> inverted_indexes;
@@ -683,6 +687,7 @@ private:
segment_v2::CompressionTypePB _compression_type =
segment_v2::CompressionTypePB::LZ4F;
long _row_store_page_size = segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
long _storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ long _storage_dict_page_size =
segment_v2::STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
size_t _next_column_unique_id = 0;
std::string _auto_increment_column;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index e5e869c393f..7306b0f0af4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -233,7 +233,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
tbl.getEnableMowLightDelete(), null,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(), true);
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 69a5734c40a..d6eed289d34 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -274,7 +274,8 @@ public class CloudSchemaChangeJobV2 extends
SchemaChangeJobV2 {
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(), true);
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index e5d97d1f97b..9ae7d3b39df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -288,7 +288,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
objectPool,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index a9065f92389..fb4a4c8f2b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -334,7 +334,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
objectPool,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index 1fd7480c06c..29ec458eb42 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -366,8 +366,9 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN))
{
// do nothing, will be analyzed when creating alter job
- } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
- throw new AnalysisException("You can not modify
storage_page_size");
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)
+ ||
properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+ throw new AnalysisException("You can not modify
storage_page_size|storage_dict_page_size");
} else {
throw new AnalysisException("Unknown table property: " +
properties.keySet());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 9cddecc5d74..0b50bd28c8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1425,7 +1425,8 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
objectPool,
localTbl.rowStorePageSize(),
localTbl.variantEnableFlattenNested(),
- localTbl.storagePageSize());
+ localTbl.storagePageSize(),
+ localTbl.storageDictPageSize());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
if (baseTabletRef != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index e42483a65bf..7b05aac0c51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3763,6 +3763,12 @@ public class Env {
sb.append(olapTable.storagePageSize()).append("\"");
}
+ // storage dict page size
+ if (olapTable.storageDictPageSize() !=
PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE) {
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE).append("\"
= \"");
+ sb.append(olapTable.storageDictPageSize()).append("\"");
+ }
+
// skip inverted index on load
if (olapTable.skipWriteIndexOnLoad()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).append("\"
= \"");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index a1b6c3a3f3f..a90a4efa485 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2772,6 +2772,20 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
}
+ public void setStorageDictPageSize(long storageDictPageSize) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
+ Long.valueOf(storageDictPageSize).toString());
+ tableProperty.buildStorageDictPageSize();
+ }
+
+ public long storageDictPageSize() {
+ if (tableProperty != null) {
+ return tableProperty.storageDictPageSize();
+ }
+ return PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+ }
+
public void setStorageFormat(TStorageFormat storageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT,
storageFormat.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index d0c7dc6ea2c..4093c115ed3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -109,6 +109,8 @@ public class TableProperty implements Writable,
GsonPostProcessable {
private long storagePageSize =
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ private long storageDictPageSize =
PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+
private String compactionPolicy =
PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY;
private long timeSeriesCompactionGoalSizeMbytes
@@ -355,6 +357,17 @@ public class TableProperty implements Writable,
GsonPostProcessable {
return storagePageSize;
}
+ public TableProperty buildStorageDictPageSize() {
+ storageDictPageSize = Long.parseLong(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
+
Long.toString(PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE)));
+ return this;
+ }
+
+ public long storageDictPageSize() {
+ return storageDictPageSize;
+ }
+
public TableProperty buildSkipWriteIndexOnLoad() {
skipWriteIndexOnLoad = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
"false"));
@@ -762,6 +775,7 @@ public class TableProperty implements Writable,
GsonPostProcessable {
buildRowStoreColumns();
buildRowStorePageSize();
buildStoragePageSize();
+ buildStorageDictPageSize();
buildSkipWriteIndexOnLoad();
buildCompactionPolicy();
buildTimeSeriesCompactionGoalSizeMbytes();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index aa6a54e1622..df0147ba51f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -383,7 +383,8 @@ public class CloudRestoreJob extends RestoreJob {
localTbl.getInvertedIndexFileStorageFormat(),
localTbl.rowStorePageSize(),
localTbl.variantEnableFlattenNested(),
- localTbl.storagePageSize(), false));
+ localTbl.storagePageSize(),
+ localTbl.storageDictPageSize(), false));
// In cloud mode all storage medium will be saved to HDD.
TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
restoredIdx.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 568f3451d5e..a889505b3b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -177,7 +177,8 @@ public class CloudInternalCatalog extends InternalCatalog {
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(), true);
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
}
requestBuilder.setDbId(dbId);
@@ -209,7 +210,8 @@ public class CloudInternalCatalog extends InternalCatalog {
Long timeSeriesCompactionLevelThreshold, boolean
disableAutoCompaction,
List<Integer> rowStoreColumnUniqueIds, boolean
enableMowLightDelete,
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat,
long pageSize,
- boolean variantEnableFlattenNested, long storagePageSize, boolean
createInitialRowset) throws DdlException {
+ boolean variantEnableFlattenNested, long storagePageSize, long
storageDictPageSize, boolean
+ createInitialRowset) throws DdlException {
OlapFile.TabletMetaCloudPB.Builder builder =
OlapFile.TabletMetaCloudPB.newBuilder();
builder.setTableId(tableId);
builder.setIndexId(indexId);
@@ -353,6 +355,7 @@ public class CloudInternalCatalog extends InternalCatalog {
}
schemaBuilder.setRowStorePageSize(pageSize);
schemaBuilder.setStoragePageSize(storagePageSize);
+ schemaBuilder.setStorageDictPageSize(storageDictPageSize);
schemaBuilder.setEnableVariantFlattenNested(variantEnableFlattenNested);
OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 62d68529d4d..071916a804b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -106,6 +106,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_STORAGE_PAGE_SIZE =
"storage_page_size";
public static final long STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536L;
+ public static final String PROPERTIES_STORAGE_DICT_PAGE_SIZE =
"storage_dict_page_size";
+ public static final long STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE = 262144L;
+
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE =
"distribution_type";
@@ -1118,6 +1121,24 @@ public class PropertyAnalyzer {
return storagePageSize;
}
+ public static long analyzeStorageDictPageSize(Map<String, String>
properties) throws AnalysisException {
+ long storageDictPageSize = STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+ if (properties != null &&
properties.containsKey(PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+ String storageDictPageSizeStr =
properties.get(PROPERTIES_STORAGE_DICT_PAGE_SIZE);
+ try {
+ storageDictPageSize = Long.parseLong(storageDictPageSizeStr);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid storage dict page size: "
+ storageDictPageSizeStr);
+ }
+ if (storageDictPageSize < 0 || storageDictPageSize > 104857600) {
+ throw new AnalysisException("Storage dict page size must be
between 0 and 100MB.");
+ }
+ storageDictPageSize = alignTo4K(storageDictPageSize);
+ properties.remove(PROPERTIES_STORAGE_DICT_PAGE_SIZE);
+ }
+ return storageDictPageSize;
+ }
+
// analyzeStorageFormat will parse the storage format from properties
// sql: alter table tablet_name set ("storage_format" = "v2")
// Use this sql to convert all tablets(base and rollup index) to a new
format segment
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index e3541bae3cf..713e8a0f772 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1670,6 +1670,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
Long.toString(olapTable.storagePageSize()));
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
+ Long.toString(olapTable.storageDictPageSize()));
+ }
if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD))
{
properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
olapTable.skipWriteIndexOnLoad().toString());
@@ -2200,7 +2204,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool, tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
@@ -2710,6 +2715,14 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
olapTable.setStoragePageSize(storagePageSize);
+ long storageDictPageSize =
PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+ try {
+ storageDictPageSize =
PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ olapTable.setStorageDictPageSize(storageDictPageSize);
+
// check data sort properties
int keyColumnSize =
CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames()) ?
keysDesc.keysColumnSize() :
keysDesc.getClusterKeysColumnNames().size();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 439ee337783..ea8dceea2b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -1081,7 +1081,8 @@ public class ReportHandler extends Daemon {
objectPool,
olapTable.rowStorePageSize(),
olapTable.variantEnableFlattenNested(),
- olapTable.storagePageSize());
+ olapTable.storagePageSize(),
+ olapTable.storageDictPageSize());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
.getInvertedIndexFileStorageFormat());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 6328bf2f2a0..4e369acf54e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -67,6 +67,7 @@ public class CreateReplicaTask extends AgentTask {
private TCompressionType compressionType;
private long rowStorePageSize;
private long storagePageSize;
+ private long storageDictPageSize;
private List<Column> columns;
@@ -158,7 +159,8 @@ public class CreateReplicaTask extends AgentTask {
Map<Object, Object> objectPool,
long rowStorePageSize,
boolean variantEnableFlattenNested,
- long storagePageSize) {
+ long storagePageSize,
+ long storageDictPageSize) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
this.replicaId = replicaId;
@@ -207,6 +209,7 @@ public class CreateReplicaTask extends AgentTask {
this.rowStorePageSize = rowStorePageSize;
this.variantEnableFlattenNested = variantEnableFlattenNested;
this.storagePageSize = storagePageSize;
+ this.storageDictPageSize = storageDictPageSize;
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -369,6 +372,7 @@ public class CreateReplicaTask extends AgentTask {
tSchema.setStoreRowColumn(storeRowColumn);
tSchema.setRowStorePageSize(rowStorePageSize);
tSchema.setStoragePageSize(storagePageSize);
+ tSchema.setStorageDictPageSize(storageDictPageSize);
createTabletReq.setTabletSchema(tSchema);
createTabletReq.setVersion(version);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 6d708aa0826..ab7291eaf16 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -191,6 +191,54 @@ public class PropertyAnalyzerTest {
Assert.assertEquals(Tag.DEFAULT_BACKEND_TAG.value,
tagMap.get(Tag.TYPE_LOCATION));
}
+ @Test
+ public void testStorageDictPageSize() throws AnalysisException {
+ Map<String, String> properties = Maps.newHashMap();
+
+ // Test default value
+
Assert.assertEquals(PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE,
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test valid value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"8192"); // 8KB
+ Assert.assertEquals(8192,
PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test lower boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"4096"); // 4KB
+ Assert.assertEquals(4096,
PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test upper boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"10485760"); // 10MB
+ Assert.assertEquals(10485760,
PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test invalid number format
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"invalid");
+ try {
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid storage dict
page size"));
+ }
+
+ // Test value below minimum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"-1024"); // 1KB
+ try {
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage dict page size
must be between 0 and 100MB"));
+ }
+
+ // Test value above maximum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"209715200"); // 200MB
+ try {
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage dict page size
must be between 0 and 100MB"));
+ }
+ }
+
@Test
public void testStoragePageSize() throws AnalysisException {
Map<String, String> properties = Maps.newHashMap();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index ce3d07c85d7..55c14a658e6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -76,6 +76,9 @@ public class AgentTaskTest {
private TStorageType storageType = TStorageType.COLUMN;
private long rowStorePageSize = 16384L;
private long storagePageSize = 65536L;
+
+ private long storageDictPageSize = 262144L;
+
private List<Column> columns;
private MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(3);
@@ -112,7 +115,7 @@ public class AgentTaskTest {
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false,
- storagePageSize);
+ storagePageSize, storageDictPageSize);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 9c3fc1b3d3e..57cab22aa37 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -424,6 +424,7 @@ message TabletSchemaPB {
optional bool variant_enable_flatten_nested = 28 [default=false];
optional int32 skip_bitmap_col_idx = 29 [default = -1];
optional int64 storage_page_size = 30 [default=65536];
+ optional int64 storage_dict_page_size = 31 [default=262144];
}
message TabletSchemaCloudPB {
@@ -458,6 +459,7 @@ message TabletSchemaCloudPB {
optional bool enable_variant_flatten_nested = 29 [default=false];
optional int32 skip_bitmap_col_idx = 30 [default = -1];
optional int64 storage_page_size = 31 [default=65536];
+ optional int64 storage_dict_page_size = 32 [default=262144];
optional bool is_dynamic_schema = 100 [default=false];
}
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index dd7fd1f0bdd..a86bd3c84ef 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -50,6 +50,7 @@ struct TTabletSchema {
21: optional i64 row_store_page_size = 16384
22: optional bool variant_enable_flatten_nested = false
23: optional i64 storage_page_size = 65536
+ 24: optional i64 storage_dict_page_size = 262144
}
// this enum stands for different storage format in src_backends
diff --git a/regression-test/data/query_p0/system/test_table_properties.out
b/regression-test/data/query_p0/system/test_table_properties.out
index 812b7d2d4b7..aa121b9a302 100644
Binary files a/regression-test/data/query_p0/system/test_table_properties.out
and b/regression-test/data/query_p0/system/test_table_properties.out differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
