This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new ccd0b93b1b4 [fix](Row store) get the correct value of
row_store_page_size after FE restarts (#38240)
ccd0b93b1b4 is described below
commit ccd0b93b1b4c26f71f01b24a1a231822a9b375ae
Author: Xr Ling <[email protected]>
AuthorDate: Thu Jul 25 10:39:18 2024 +0800
[fix](Row store) get the correct value of row_store_page_size after FE
restarts (#38240)
pick #38236
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 5 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 5 +-
.../org/apache/doris/catalog/TableProperty.java | 1 +
.../cloud/datasource/CloudInternalCatalog.java | 2 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 2 +-
.../apache/doris/datasource/InternalCatalog.java | 12 +--
gensrc/thrift/AgentService.thrift | 2 +-
.../test_row_store_page_size.out | 10 +++
.../data/query_p0/system/test_table_options.out | 9 ---
.../test_row_store_page_size.groovy | 90 ++++++++++++++++++++++
10 files changed, 118 insertions(+), 20 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 5588bd79f17..bdfcaba8b8e 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -258,8 +258,11 @@ Status SegmentWriter::_create_column_writer(uint32_t cid,
const TabletColumn& co
if (column.is_row_store_column()) {
// smaller page size for row store column
- opts.data_page_size = _tablet_schema->row_store_page_size();
+ auto page_size = _tablet_schema->row_store_page_size();
+ opts.data_page_size =
+ (page_size > 0) ? page_size :
segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
}
+
std::unique_ptr<ColumnWriter> writer;
RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer,
&writer));
RETURN_IF_ERROR(writer->init());
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 0307a53d740..ba1bfcf3535 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -221,8 +221,11 @@ Status
VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
if (column.is_row_store_column()) {
// smaller page size for row store column
- opts.data_page_size = _tablet_schema->row_store_page_size();
+ auto page_size = _tablet_schema->row_store_page_size();
+ opts.data_page_size =
+ (page_size > 0) ? page_size :
segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
}
+
std::unique_ptr<ColumnWriter> writer;
RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer,
&writer));
RETURN_IF_ERROR(writer->init());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 21f96a202f1..5ac446411f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -670,6 +670,7 @@ public class TableProperty implements Writable,
GsonPostProcessable {
buildEnableLightSchemaChange();
buildStoreRowColumn();
buildRowStoreColumns();
+ buildRowStorePageSize();
buildSkipWriteIndexOnLoad();
buildCompactionPolicy();
buildTimeSeriesCompactionGoalSizeMbytes();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 20b1e17f529..0e9cbf1128b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -102,7 +102,7 @@ public class CloudInternalCatalog extends InternalCatalog {
IdGeneratorBuffer
idGeneratorBuffer,
BinlogConfig binlogConfig,
boolean
isStorageMediumSpecified,
- List<Integer>
clusterKeyIndexes, long pageSize)
+ List<Integer>
clusterKeyIndexes)
throws DdlException {
// create base index first.
Preconditions.checkArgument(tbl.getBaseIndexId() != -1);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index e339d8e5e11..3d1ab7c8951 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -97,7 +97,7 @@ public class PropertyAnalyzer {
// row store page size, default 16KB
public static final String PROPERTIES_ROW_STORE_PAGE_SIZE =
"row_store_page_size";
- public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384;
+ public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384L;
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index c4ebad14425..0bb64cc02c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1710,7 +1710,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
singlePartitionDesc.isInMemory(),
singlePartitionDesc.getTabletType(),
storagePolicy, idGeneratorBuffer,
- binlogConfig, dataProperty.isStorageMediumSpecified(),
null, olapTable.rowStorePageSize());
+ binlogConfig, dataProperty.isStorageMediumSpecified(),
null);
// TODO cluster key ids
// check again
@@ -2006,7 +2006,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
IdGeneratorBuffer
idGeneratorBuffer,
BinlogConfig binlogConfig,
boolean
isStorageMediumSpecified,
- List<Integer>
clusterKeyIndexes, long rowStorePageSize)
+ List<Integer>
clusterKeyIndexes)
throws DdlException {
// create base index first.
Preconditions.checkArgument(tbl.getBaseIndexId() != -1);
@@ -2090,7 +2090,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.getTimeSeriesCompactionLevelThreshold(),
tbl.storeRowColumn(), binlogConfig,
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
- objectPool, rowStorePageSize);
+ objectPool, tbl.rowStorePageSize());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
@@ -2880,7 +2880,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
idGeneratorBuffer,
binlogConfigForTask,
partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(),
- keysDesc.getClusterKeysColumnIds(),
olapTable.rowStorePageSize());
+ keysDesc.getClusterKeysColumnIds());
afterCreatePartitions(db.getId(), olapTable.getId(), null,
olapTable.getIndexIdList(), true);
olapTable.addPartition(partition);
@@ -2963,7 +2963,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
partionStoragePolicy, idGeneratorBuffer,
binlogConfigForTask,
dataProperty.isStorageMediumSpecified(),
- keysDesc.getClusterKeysColumnIds(),
olapTable.rowStorePageSize());
+ keysDesc.getClusterKeysColumnIds());
olapTable.addPartition(partition);
olapTable.getPartitionInfo().getDataProperty(partition.getId())
.setStoragePolicy(partionStoragePolicy);
@@ -3430,7 +3430,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(),
idGeneratorBuffer, binlogConfig,
copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(),
- clusterKeyIdxes, olapTable.rowStorePageSize());
+ clusterKeyIdxes);
newPartitions.add(newPartition);
}
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 8d24e64c018..767ede9017a 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -47,7 +47,7 @@ struct TTabletSchema {
19: optional list<i32> cluster_key_idxes
// col unique id for row store column
20: optional list<i32> row_store_col_cids
- 21: optional i64 row_store_page_size = 16384;
+ 21: optional i64 row_store_page_size = 16384
}
// this enum stands for different storage format in src_backends
diff --git
a/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out
b/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out
new file mode 100644
index 00000000000..95b3ccd4950
--- /dev/null
+++
b/regression-test/data/cloud_p0/row_store_page_size/test_row_store_page_size.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_star --
+1 1 1 a
+
+-- !select_star --
+3 3 \N c
+
+-- !select_star --
+1 1 1 a
+
diff --git a/regression-test/data/query_p0/system/test_table_options.out
b/regression-test/data/query_p0/system/test_table_options.out
index 34a5a27c301..66856b811fb 100644
--- a/regression-test/data/query_p0/system/test_table_options.out
+++ b/regression-test/data/query_p0/system/test_table_options.out
@@ -1,6 +1,5 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select --
-<<<<<<< HEAD
aggregate_table internal test_table_options_db AGG
user_id,date,city,age,sex user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_com [...]
duplicate_table internal test_table_options_db DUP
timestamp,type,error_code type HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compac [...]
listtable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 16 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_serie [...]
@@ -9,12 +8,4 @@ rangetable internal test_table_options_db AGG
user_id,date,timestamp,city,age,se
test_row_column_page_size1 internal test_table_options_db DUP
aaa aaa HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_t [...]
test_row_column_page_size2 internal test_table_options_db DUP
aaa aaa HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"1","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_t [...]
unique_table internal test_table_options_db UNI
user_id,username user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_leve [...]
-=======
-aggregate_table internal test_table_options_db AGG
user_id,date,city,age,sex user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_com [...]
-duplicate_table internal test_table_options_db DUP
timestamp,type,error_code type HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compac [...]
-listtable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 16 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_serie [...]
-randomtable internal test_table_options_db DUP
user_id,date,timestamp RANDOM RANDOM 16 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compacti [...]
-rangetable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 8 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_serie [...]
-unique_table internal test_table_options_db UNI
user_id,username user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_leve [...]
->>>>>>> 2f3a9c43ad ([Fix](paritial update) Fix the case of partial update
failing in cloud mode (#37151))
diff --git
a/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy
b/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy
new file mode 100644
index 00000000000..4be53ff17f6
--- /dev/null
+++
b/regression-test/suites/cloud_p0/row_store_page_size/test_row_store_page_size.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_row_store_page_size_cloud") {
+
+ sql """ DROP TABLE IF EXISTS ps_table_1; """
+
+ sql """
+ create table ps_table_1(
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ unique key (k1,k2)
+ distributed BY hash(k1) buckets 3
+ properties("replication_num" = "1", "store_row_column" = "true");
+ """
+
+ test {
+ sql "show create table ps_table_1;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"row_store_page_size\" =
\"16384\""))
+ }
+ }
+
+ sql "insert into ps_table_1 select 1,1,1,'a';"
+ sql "insert into ps_table_1 select 2,2,2,'b';"
+ sql "insert into ps_table_1 select 3,3,null,'c';"
+
+ explain {
+ sql("select * from ps_table_1 where k1=1 and k2=1;")
+ contains("SHORT")
+ }
+
+ qt_select_star "select * from ps_table_1 where k1=1 and k2=1;"
+ qt_select_star "select * from ps_table_1 where k1=3 and k2=3;"
+
+ sql """ DROP TABLE IF EXISTS ps_table_1; """
+
+ sql """ DROP TABLE IF EXISTS ps_table_2; """
+
+ sql """
+ create table ps_table_2(
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ unique key (k1,k2)
+ distributed BY hash(k1) buckets 3
+ properties("replication_num" = "1", "store_row_column" = "true",
"row_store_page_size" = "8190");
+ """
+
+ test {
+ sql "show create table ps_table_2;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"row_store_page_size\" =
\"8192\""))
+ }
+ }
+
+ sql "insert into ps_table_2 select 1,1,1,'a';"
+ sql "insert into ps_table_2 select 2,2,2,'b';"
+ sql "insert into ps_table_2 select 3,3,null,'c';"
+
+ explain {
+ sql("select * from ps_table_2 where k1=1 and k2=1;")
+ contains("SHORT")
+ }
+
+ qt_select_star "select * from ps_table_2 where k1=1 and k2=1;"
+
+ sql """ DROP TABLE IF EXISTS ps_table_2; """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
