This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ee27bdb6e42 [opt](store) allow to set storage_page_size for tables
(#43690)
ee27bdb6e42 is described below
commit ee27bdb6e42ddf659cac1e8c1c32b7811b4cea99
Author: zzzxl <[email protected]>
AuthorDate: Wed Nov 13 14:35:46 2024 +0800
[opt](store) allow to set storage_page_size for tables (#43690)
Problem Summary:
1. The storage_page_size can be set within a range of 4k to 1M.
---
be/src/cloud/pb_convert.cpp | 4 +
be/src/olap/rowset/segment_v2/options.h | 1 +
be/src/olap/rowset/segment_v2/segment_writer.cpp | 27 +++
.../rowset/segment_v2/vertical_segment_writer.cpp | 27 +++
be/src/olap/tablet_meta.cpp | 3 +
be/src/olap/tablet_schema.cpp | 4 +
be/src/olap/tablet_schema.h | 3 +
.../org/apache/doris/alter/CloudRollupJobV2.java | 3 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../analysis/ModifyTablePropertiesClause.java | 2 +
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 6 +
.../java/org/apache/doris/catalog/OlapTable.java | 14 ++
.../org/apache/doris/catalog/TableProperty.java | 13 ++
.../cloud/datasource/CloudInternalCatalog.java | 7 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 21 +++
.../apache/doris/datasource/InternalCatalog.java | 15 +-
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../org/apache/doris/task/CreateReplicaTask.java | 6 +-
.../apache/doris/common/PropertyAnalyzerTest.java | 48 ++++++
.../java/org/apache/doris/task/AgentTaskTest.java | 4 +-
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 1 +
.../data/query_p0/system/test_table_properties.out | 8 +-
.../test_storage_page_size_fault.groovy | 71 ++++++++
.../query_p0/system/test_storage_page_size.groovy | 190 +++++++++++++++++++++
28 files changed, 483 insertions(+), 12 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 1f780824e32..bff7d8388d3 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -330,6 +330,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
const TabletSchemaPB
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_page_size(in.storage_page_size());
}
void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&&
in) {
@@ -358,6 +359,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
TabletSchemaPB&& in)
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_page_size(in.storage_page_size());
}
TabletSchemaPB cloud_tablet_schema_to_doris(const TabletSchemaCloudPB& in) {
@@ -399,6 +401,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
const TabletSchemaCloudPB
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_page_size(in.storage_page_size());
}
void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&&
in) {
@@ -428,6 +431,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
TabletSchemaCloudPB&& in)
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
+ out->set_storage_page_size(in.storage_page_size());
}
TabletMetaCloudPB doris_tablet_meta_to_cloud(const TabletMetaPB& in) {
diff --git a/be/src/olap/rowset/segment_v2/options.h
b/be/src/olap/rowset/segment_v2/options.h
index 93ec03df452..164a8a27fc1 100644
--- a/be/src/olap/rowset/segment_v2/options.h
+++ b/be/src/olap/rowset/segment_v2/options.h
@@ -23,6 +23,7 @@ namespace doris {
namespace segment_v2 {
static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M
+static constexpr size_t STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536;
constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store
page size: 16KB
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index e4288338d48..09ff3f6ed3b 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -247,6 +247,33 @@ Status SegmentWriter::_create_column_writer(uint32_t cid,
const TabletColumn& co
#undef DISABLE_INDEX_IF_FIELD_TYPE
+ int64_t storage_page_size = _tablet_schema->storage_page_size();
+ // storage_page_size must be between 4KB and 10MB.
+ if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
+ opts.data_page_size = storage_page_size;
+ }
+
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
+ auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
+ INT_MIN);
+ auto target_data_page_size =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size",
+ "storage_page_size", INT_MIN);
+ if (table_id == INT_MIN || target_data_page_size == INT_MIN) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Debug point parameters missing: either 'table_id' or
'storage_page_size' not "
+ "set.");
+ }
+ if (table_id == _tablet_schema->table_id() &&
+ opts.data_page_size != target_data_page_size) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Mismatch in 'storage_page_size': expected size does not
match the current "
+ "data page size. "
+ "Expected: " +
+ std::to_string(target_data_page_size) +
+ ", Actual: " + std::to_string(opts.data_page_size) + ".");
+ }
+ })
if (column.is_row_store_column()) {
// smaller page size for row store column
auto page_size = _tablet_schema->row_store_page_size();
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index e8804d93e78..12028812f0d 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -242,6 +242,33 @@ Status
VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
#undef CHECK_FIELD_TYPE
+ int64_t storage_page_size = _tablet_schema->storage_page_size();
+ // storage_page_size must be between 4KB and 10MB.
+ if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
+ opts.data_page_size = storage_page_size;
+ }
+
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
+ auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
+ INT_MIN);
+ auto target_data_page_size =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size",
+ "storage_page_size", INT_MIN);
+ if (table_id == INT_MIN || target_data_page_size == INT_MIN) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Debug point parameters missing: either 'table_id' or
'storage_page_size' not "
+ "set.");
+ }
+ if (table_id == _tablet_schema->table_id() &&
+ opts.data_page_size != target_data_page_size) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Mismatch in 'storage_page_size': expected size does not
match the current "
+ "data page size. "
+ "Expected: " +
+ std::to_string(target_data_page_size) +
+ ", Actual: " + std::to_string(opts.data_page_size) + ".");
+ }
+ })
if (column.is_row_store_column()) {
// smaller page size for row store column
auto page_size = _tablet_schema->row_store_page_size();
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 91f3b7dd816..13bbdaa9389 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -329,6 +329,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
if (tablet_schema.__isset.row_store_page_size) {
schema->set_row_store_page_size(tablet_schema.row_store_page_size);
}
+ if (tablet_schema.__isset.storage_page_size) {
+ schema->set_storage_page_size(tablet_schema.storage_page_size);
+ }
if (tablet_schema.__isset.skip_write_index_on_load) {
schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 523944c4d77..c4f96e22148 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1043,6 +1043,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_sort_col_num = schema.sort_col_num();
_compression_type = schema.compression_type();
_row_store_page_size = schema.row_store_page_size();
+ _storage_page_size = schema.storage_page_size();
_schema_version = schema.schema_version();
// Default to V1 inverted index storage format for backward compatibility
if not specified in schema.
if (!schema.has_inverted_index_storage_format()) {
@@ -1107,6 +1108,7 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
_sort_type = ori_tablet_schema.sort_type();
_sort_col_num = ori_tablet_schema.sort_col_num();
_row_store_page_size = ori_tablet_schema.row_store_page_size();
+ _storage_page_size = ori_tablet_schema.storage_page_size();
_variant_enable_flatten_nested =
ori_tablet_schema.variant_flatten_nested();
// copy from table_schema_param
@@ -1266,6 +1268,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
tablet_schema_pb->set_schema_version(_schema_version);
tablet_schema_pb->set_compression_type(_compression_type);
tablet_schema_pb->set_row_store_page_size(_row_store_page_size);
+ tablet_schema_pb->set_storage_page_size(_storage_page_size);
tablet_schema_pb->set_version_col_idx(_version_col_idx);
tablet_schema_pb->set_skip_bitmap_col_idx(_skip_bitmap_col_idx);
tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format);
@@ -1533,6 +1536,7 @@ bool operator==(const TabletSchema& a, const
TabletSchema& b) {
if (a._enable_single_replica_compaction !=
b._enable_single_replica_compaction) return false;
if (a._store_row_column != b._store_row_column) return false;
if (a._row_store_page_size != b._row_store_page_size) return false;
+ if (a._storage_page_size != b._storage_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return
false;
if (a._variant_enable_flatten_nested != b._variant_enable_flatten_nested)
return false;
return true;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 1e66e36809d..5fb3deafd77 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -394,6 +394,8 @@ public:
segment_v2::CompressionTypePB compression_type() const { return
_compression_type; }
void set_row_store_page_size(long page_size) { _row_store_page_size =
page_size; }
long row_store_page_size() const { return _row_store_page_size; }
+ void set_storage_page_size(long storage_page_size) { _storage_page_size =
storage_page_size; }
+ long storage_page_size() const { return _storage_page_size; }
const std::vector<const TabletIndex*> inverted_indexes() const {
std::vector<const TabletIndex*> inverted_indexes;
@@ -555,6 +557,7 @@ private:
CompressKind _compress_kind = COMPRESS_NONE;
segment_v2::CompressionTypePB _compression_type =
segment_v2::CompressionTypePB::LZ4F;
long _row_store_page_size = segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+ long _storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE;
size_t _next_column_unique_id = 0;
std::string _auto_increment_column;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index b38921f4fc9..37638319e3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -232,7 +232,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
tbl.getEnableMowLightDelete(), null,
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(), null);
+ tbl.variantEnableFlattenNested(), null,
+ tbl.storagePageSize());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 07b8323951d..92a74e46b92 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -277,7 +277,8 @@ public class CloudSchemaChangeJobV2 extends
SchemaChangeJobV2 {
tbl.getEnableMowLightDelete(),
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(),
clusterKeyIndexes);
+ tbl.variantEnableFlattenNested(),
clusterKeyIndexes,
+ tbl.storagePageSize());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 49838446cd5..8eb87050bf6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -270,7 +270,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
tbl.getRowStoreColumnsUniqueIds(tbl.getTableProperty().getCopiedRowStoreColumns()),
objectPool,
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested());
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index c8754aa5d62..522433dfe89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -330,7 +330,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool,
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested());
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index 71acd3b217b..1fd7480c06c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -366,6 +366,8 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN))
{
// do nothing, will be analyzed when creating alter job
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
+ throw new AnalysisException("You can not modify
storage_page_size");
} else {
throw new AnalysisException("Unknown table property: " +
properties.keySet());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 203e7f82a73..aef09bd78be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1353,7 +1353,8 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool,
localTbl.rowStorePageSize(),
- localTbl.variantEnableFlattenNested());
+ localTbl.variantEnableFlattenNested(),
+ localTbl.storagePageSize());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
if (baseTabletRef != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index f67444b6cb1..c7b3d350c36 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3612,6 +3612,12 @@ public class Env {
sb.append(olapTable.rowStorePageSize()).append("\"");
}
+ // storage page size
+ if (olapTable.storagePageSize() !=
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE) {
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE).append("\"
= \"");
+ sb.append(olapTable.storagePageSize()).append("\"");
+ }
+
// skip inverted index on load
if (olapTable.skipWriteIndexOnLoad()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).append("\"
= \"");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 082fa71060d..8a9ce4a2a2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2764,6 +2764,20 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
}
+ public void setStoragePageSize(long storagePageSize) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
+ Long.valueOf(storagePageSize).toString());
+ tableProperty.buildStoragePageSize();
+ }
+
+ public long storagePageSize() {
+ if (tableProperty != null) {
+ return tableProperty.storagePageSize();
+ }
+ return PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ }
+
public void setStorageFormat(TStorageFormat storageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT,
storageFormat.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 8517c4fa58a..25cf2f4287e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -107,6 +107,8 @@ public class TableProperty implements Writable,
GsonPostProcessable {
private long rowStorePageSize =
PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+ private long storagePageSize =
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+
private String compactionPolicy =
PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY;
private long timeSeriesCompactionGoalSizeMbytes
@@ -324,6 +326,17 @@ public class TableProperty implements Writable,
GsonPostProcessable {
return rowStorePageSize;
}
+ public TableProperty buildStoragePageSize() {
+ storagePageSize = Long.parseLong(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
+
Long.toString(PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE)));
+ return this;
+ }
+
+ public long storagePageSize() {
+ return storagePageSize;
+ }
+
public TableProperty buildSkipWriteIndexOnLoad() {
skipWriteIndexOnLoad = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
"false"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index fab1ce58775..b8a364f9449 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -180,7 +180,8 @@ public class CloudInternalCatalog extends InternalCatalog {
tbl.getEnableMowLightDelete(),
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(), clusterKeyIndexes);
+ tbl.variantEnableFlattenNested(), clusterKeyIndexes,
+ tbl.storagePageSize());
requestBuilder.addTabletMetas(builder);
}
if (!storageVaultIdSet && ((CloudEnv)
Env.getCurrentEnv()).getEnableStorageVault()) {
@@ -230,7 +231,8 @@ public class CloudInternalCatalog extends InternalCatalog {
Long timeSeriesCompactionLevelThreshold, boolean
disableAutoCompaction,
List<Integer> rowStoreColumnUniqueIds, boolean
enableMowLightDelete,
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat,
long pageSize,
- boolean variantEnableFlattenNested, List<Integer> clusterKeyIdxes)
throws DdlException {
+ boolean variantEnableFlattenNested, List<Integer> clusterKeyIdxes,
+ long storagePageSize) throws DdlException {
OlapFile.TabletMetaCloudPB.Builder builder =
OlapFile.TabletMetaCloudPB.newBuilder();
builder.setTableId(tableId);
builder.setIndexId(indexId);
@@ -357,6 +359,7 @@ public class CloudInternalCatalog extends InternalCatalog {
}
}
schemaBuilder.setRowStorePageSize(pageSize);
+ schemaBuilder.setStoragePageSize(storagePageSize);
schemaBuilder.setEnableVariantFlattenNested(variantEnableFlattenNested);
if (!CollectionUtils.isEmpty(clusterKeyIdxes)) {
schemaBuilder.addAllClusterKeyIdxes(clusterKeyIdxes);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index b4384f8a7fa..5721db0c27e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -99,6 +99,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_ROW_STORE_PAGE_SIZE =
"row_store_page_size";
public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384L;
+ public static final String PROPERTIES_STORAGE_PAGE_SIZE =
"storage_page_size";
+ public static final long STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536L;
+
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE =
"distribution_type";
@@ -1072,6 +1075,24 @@ public class PropertyAnalyzer {
return rowStorePageSize;
}
+ public static long analyzeStoragePageSize(Map<String, String> properties)
throws AnalysisException {
+ long storagePageSize = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ if (properties != null &&
properties.containsKey(PROPERTIES_STORAGE_PAGE_SIZE)) {
+ String storagePageSizeStr =
properties.get(PROPERTIES_STORAGE_PAGE_SIZE);
+ try {
+ storagePageSize = Long.parseLong(storagePageSizeStr);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid storage page size: " +
storagePageSizeStr);
+ }
+ if (storagePageSize < 4096 || storagePageSize > 10485760) {
+ throw new AnalysisException("Storage page size must be between
4KB and 10MB.");
+ }
+ storagePageSize = alignTo4K(storagePageSize);
+ properties.remove(PROPERTIES_STORAGE_PAGE_SIZE);
+ }
+ return storagePageSize;
+ }
+
// analyzeStorageFormat will parse the storage format from properties
// sql: alter table tablet_name set ("storage_format" = "v2")
// Use this sql to convert all tablets(base and rollup index) to a new
format segment
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index f8183028c6a..8dbfa305bf4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1641,6 +1641,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
properties.put(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE,
Long.toString(olapTable.rowStorePageSize()));
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
+ Long.toString(olapTable.storagePageSize()));
+ }
if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD))
{
properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
olapTable.skipWriteIndexOnLoad().toString());
@@ -2168,7 +2172,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.storeRowColumn(), binlogConfig,
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool, tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested());
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
@@ -2658,6 +2663,14 @@ public class InternalCatalog implements
CatalogIf<Database> {
olapTable.setRowStorePageSize(rowStorePageSize);
+ long storagePageSize =
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ try {
+ storagePageSize =
PropertyAnalyzer.analyzeStoragePageSize(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ olapTable.setStoragePageSize(storagePageSize);
+
// check data sort properties
int keyColumnSize =
CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames()) ?
keysDesc.keysColumnSize() :
keysDesc.getClusterKeysColumnNames().size();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index bc51590ccb5..c5c72eae3c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -974,7 +974,8 @@ public class ReportHandler extends Daemon {
olapTable.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool,
olapTable.rowStorePageSize(),
-
olapTable.variantEnableFlattenNested());
+
olapTable.variantEnableFlattenNested(),
+ olapTable.storagePageSize());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
.getInvertedIndexFileStorageFormat());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index c410f37e5c9..94b73ef2864 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -66,6 +66,7 @@ public class CreateReplicaTask extends AgentTask {
private TStorageMedium storageMedium;
private TCompressionType compressionType;
private long rowStorePageSize;
+ private long storagePageSize;
private List<Column> columns;
@@ -156,7 +157,8 @@ public class CreateReplicaTask extends AgentTask {
List<Integer> rowStoreColumnUniqueIds,
Map<Object, Object> objectPool,
long rowStorePageSize,
- boolean variantEnableFlattenNested) {
+ boolean variantEnableFlattenNested,
+ long storagePageSize) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
this.replicaId = replicaId;
@@ -204,6 +206,7 @@ public class CreateReplicaTask extends AgentTask {
this.objectPool = objectPool;
this.rowStorePageSize = rowStorePageSize;
this.variantEnableFlattenNested = variantEnableFlattenNested;
+ this.storagePageSize = storagePageSize;
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -365,6 +368,7 @@ public class CreateReplicaTask extends AgentTask {
tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad);
tSchema.setStoreRowColumn(storeRowColumn);
tSchema.setRowStorePageSize(rowStorePageSize);
+ tSchema.setStoragePageSize(storagePageSize);
createTabletReq.setTabletSchema(tSchema);
createTabletReq.setVersion(version);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index eb3500d13d9..041ca89bfc5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -188,4 +188,52 @@ public class PropertyAnalyzerTest {
Assert.assertEquals(1, tagMap.size());
Assert.assertEquals(Tag.DEFAULT_BACKEND_TAG.value,
tagMap.get(Tag.TYPE_LOCATION));
}
+
+ @Test
+ public void testStoragePageSize() throws AnalysisException {
+ Map<String, String> properties = Maps.newHashMap();
+
+ // Test default value
+ Assert.assertEquals(PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE,
+ PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test valid value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "8192");
// 8KB
+ Assert.assertEquals(8192,
PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test lower boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "4096");
// 4KB
+ Assert.assertEquals(4096,
PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test upper boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
"10485760"); // 10MB
+ Assert.assertEquals(10485760,
PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test invalid number format
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
"invalid");
+ try {
+ PropertyAnalyzer.analyzeStoragePageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid storage page
size"));
+ }
+
+ // Test value below minimum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "1024");
// 1KB
+ try {
+ PropertyAnalyzer.analyzeStoragePageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage page size must
be between 4KB and 10MB"));
+ }
+
+ // Test value above maximum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
"20971520"); // 20MB
+ try {
+ PropertyAnalyzer.analyzeStoragePageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage page size must
be between 4KB and 10MB"));
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index 6f2996207d0..f6c6201a934 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -74,6 +74,7 @@ public class AgentTaskTest {
private TStorageType storageType = TStorageType.COLUMN;
private long rowStorePageSize = 16384L;
+ private long storagePageSize = 65536L;
private List<Column> columns;
private MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(3);
@@ -108,7 +109,8 @@ public class AgentTaskTest {
createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId,
partitionId,
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
- TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false);
+ TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false,
+ storagePageSize);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 30f74af7d67..259f9f2861a 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -399,6 +399,7 @@ message TabletSchemaPB {
optional bool variant_enable_flatten_nested = 28 [default=false];
optional int32 skip_bitmap_col_idx = 29 [default = -1];
+ optional int64 storage_page_size = 30 [default=65536];
}
message TabletSchemaCloudPB {
@@ -431,6 +432,7 @@ message TabletSchemaCloudPB {
optional bool enable_mow_light_delete = 28 [default=false];
optional bool enable_variant_flatten_nested = 29 [default=false];
optional int32 skip_bitmap_col_idx = 30 [default = -1];
+ optional int64 storage_page_size = 31 [default=65536];
optional bool is_dynamic_schema = 100 [default=false];
}
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index f02b8c0f886..abffd176ef8 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -49,6 +49,7 @@ struct TTabletSchema {
20: optional list<i32> row_store_col_cids
21: optional i64 row_store_page_size = 16384
22: optional bool variant_enable_flatten_nested = false
+ 23: optional i64 storage_page_size = 65536
}
// this enum stands for different storage format in src_backends
diff --git a/regression-test/data/query_p0/system/test_table_properties.out
b/regression-test/data/query_p0/system/test_table_properties.out
index 276ca4daeb5..812b7d2d4b7 100644
--- a/regression-test/data/query_p0/system/test_table_properties.out
+++ b/regression-test/data/query_p0/system/test_table_properties.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select_check_1 --
-102
+105
-- !select_check_2 --
internal test_table_properties_db duplicate_table _auto_bucket
false
@@ -29,6 +29,7 @@ internal test_table_properties_db duplicate_table
row_store_page_size 16384
internal test_table_properties_db duplicate_table
skip_write_index_on_load false
internal test_table_properties_db duplicate_table storage_format
V2
internal test_table_properties_db duplicate_table storage_medium
HDD
+internal test_table_properties_db duplicate_table
storage_page_size 65536
internal test_table_properties_db duplicate_table
store_row_column false
internal test_table_properties_db duplicate_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db duplicate_table
time_series_compaction_file_count_threshold 2000
@@ -62,6 +63,7 @@ internal test_table_properties_db listtable
row_store_page_size 16384
internal test_table_properties_db listtable
skip_write_index_on_load false
internal test_table_properties_db listtable storage_format
V2
internal test_table_properties_db listtable storage_medium
HDD
+internal test_table_properties_db listtable
storage_page_size 65536
internal test_table_properties_db listtable
store_row_column false
internal test_table_properties_db listtable
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db listtable
time_series_compaction_file_count_threshold 2000
@@ -95,6 +97,7 @@ internal test_table_properties_db unique_table
row_store_page_size 16384
internal test_table_properties_db unique_table
skip_write_index_on_load false
internal test_table_properties_db unique_table storage_format
V2
internal test_table_properties_db unique_table storage_medium
HDD
+internal test_table_properties_db unique_table
storage_page_size 65536
internal test_table_properties_db unique_table
store_row_column false
internal test_table_properties_db unique_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db unique_table
time_series_compaction_file_count_threshold 2000
@@ -130,6 +133,7 @@ internal test_table_properties_db duplicate_table
row_store_page_size 16384
internal test_table_properties_db duplicate_table
skip_write_index_on_load false
internal test_table_properties_db duplicate_table storage_format
V2
internal test_table_properties_db duplicate_table storage_medium
HDD
+internal test_table_properties_db duplicate_table
storage_page_size 65536
internal test_table_properties_db duplicate_table
store_row_column false
internal test_table_properties_db duplicate_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db duplicate_table
time_series_compaction_file_count_threshold 2000
@@ -163,6 +167,7 @@ internal test_table_properties_db unique_table
row_store_page_size 16384
internal test_table_properties_db unique_table
skip_write_index_on_load false
internal test_table_properties_db unique_table storage_format
V2
internal test_table_properties_db unique_table storage_medium
HDD
+internal test_table_properties_db unique_table
storage_page_size 65536
internal test_table_properties_db unique_table
store_row_column false
internal test_table_properties_db unique_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db unique_table
time_series_compaction_file_count_threshold 2000
@@ -200,6 +205,7 @@ internal test_table_properties_db duplicate_table
row_store_page_size 16384
internal test_table_properties_db duplicate_table
skip_write_index_on_load false
internal test_table_properties_db duplicate_table storage_format
V2
internal test_table_properties_db duplicate_table storage_medium
HDD
+internal test_table_properties_db duplicate_table
storage_page_size 65536
internal test_table_properties_db duplicate_table
store_row_column false
internal test_table_properties_db duplicate_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db duplicate_table
time_series_compaction_file_count_threshold 2000
diff --git
a/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy
b/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy
new file mode 100644
index 00000000000..76b666ecba7
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_storage_page_size_fault", "nonConcurrent") {
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ boolean disableAutoCompaction = false
+
+ def set_be_config = { key, value ->
+ for (String backend_id: backendId_to_backendIP.keySet()) {
+ def (code, out, err) =
update_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ",
err=" + err)
+ }
+ }
+
+ def dbName = "regression_test_fault_injection_p0"
+ def tableName = "test_storage_page_size_fault"
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `@timestamp` int(11) NULL COMMENT "",
+ `clientip` varchar(20) NULL COMMENT "",
+ `request` text NULL COMMENT "",
+ `status` int(11) NULL COMMENT "",
+ `size` int(11) NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`@timestamp`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true",
+ "storage_page_size" = "65537"
+ );
+ """
+
+ def tableId = getTableId(dbName, tableName)
+ if (tableId == null) {
+ throw new IllegalStateException("Table ID not found for table:
${tableName}")
+ }
+ logger.info("tableId: " + tableId)
+
+ try {
+
GetDebugPoint().enableDebugPointForAllBEs("VerticalSegmentWriter._create_column_writer.storage_page_size",
["table_id": tableId, "storage_page_size": 69632])
+ sql """ INSERT INTO ${tableName} VALUES (893964617, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+
+ set_be_config.call("enable_vertical_segment_writer", "false")
+ sql """ INSERT INTO ${tableName} VALUES (893964617, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+ set_be_config.call("enable_vertical_segment_writer", "true")
+
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("VerticalSegmentWriter._create_column_writer.storage_page_size")
+ }
+}
\ No newline at end of file
diff --git
a/regression-test/suites/query_p0/system/test_storage_page_size.groovy
b/regression-test/suites/query_p0/system/test_storage_page_size.groovy
new file mode 100644
index 00000000000..9d7b129226d
--- /dev/null
+++ b/regression-test/suites/query_p0/system/test_storage_page_size.groovy
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_storage_page_size") {
+ sql """ DROP TABLE IF EXISTS table_1; """
+ sql """
+ create table table_1 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1");
+ """
+ test {
+ sql "show create table table_1;"
+ check { result, exception, startTime, endTime ->
+ assertFalse(result[0][1].contains("\"storage_page_size\" =
\"65536\""))
+ }
+ }
+
+ // min - 1
+ sql """ DROP TABLE IF EXISTS table_2; """
+ test {
+ sql """
+ create table table_2 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "4095");
+ """
+ check { result, exception, startTime, endTime ->
+ if (exception) {
+ assert exception.message.contains("Storage page size must be
between 4KB and 10MB.")
+ }
+ }
+ }
+
+ // min
+ sql """ DROP TABLE IF EXISTS table_3; """
+ sql """
+ create table table_3 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "4096");
+ """
+ test {
+ sql "show create table table_3;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"4096\""))
+ }
+ }
+
+
+ // min + 1
+ sql """ DROP TABLE IF EXISTS table_4; """
+ sql """
+ create table table_4 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "4097");
+ """
+ test {
+ sql "show create table table_4;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"8192\""))
+ }
+ }
+
+ // 65537
+ sql """ DROP TABLE IF EXISTS table_5; """
+ sql """
+ create table table_5 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "65537");
+ """
+ test {
+ sql "show create table table_5;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"69632\""))
+ }
+ }
+ test {
+ sql """ alter table table_5 set ("storage_page_size" = "65535"); """
+ check { result, exception, startTime, endTime ->
+ if (exception) {
+ assert exception.message.contains("You can not modify
storage_page_size")
+ }
+ }
+ }
+
+ // max - 1
+ sql """ DROP TABLE IF EXISTS table_6; """
+ sql """
+ create table table_6 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "10485759");
+ """
+ test {
+ sql "show create table table_6;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"10485760\""))
+ }
+ }
+
+ // max
+ sql """ DROP TABLE IF EXISTS table_7; """
+ sql """
+ create table table_7 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "10485760");
+ """
+ test {
+ sql "show create table table_7;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"10485760\""))
+ }
+ }
+
+ // max + 1
+ sql """ DROP TABLE IF EXISTS table_8; """
+ test {
+ sql """
+ create table table_8 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" =
"10485761");
+ """
+ check { result, exception, startTime, endTime ->
+ if (exception) {
+ assert exception.message.contains("Storage page size must be
between 4KB and 10MB.")
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
