This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b49a83d145e [Update](inverted index) set default inverted index
storage format to v2 (#35303)
b49a83d145e is described below
commit b49a83d145e33a3560b08a42af58aa11ca75232d
Author: airborne12 <[email protected]>
AuthorDate: Thu Jun 6 10:41:28 2024 +0800
[Update](inverted index) set default inverted index storage format to v2
(#35303)
This pull request changes the default storage format for inverted
indexes to V2. The V2 format enhances efficiency by merging all index
files within a single segment into one file. This update aims to reduce
IO cost when writing inverted index files, especially for cloud mode.
---
.../olap/rowset/segment_v2/inverted_index_desc.cpp | 8 ++---
.../olap/rowset/segment_v2/inverted_index_desc.h | 5 +--
.../segment_v2/inverted_index_file_reader.cpp | 2 +-
.../segment_v2/inverted_index_file_writer.cpp | 39 ++++++++++----------
.../main/java/org/apache/doris/common/Config.java | 2 +-
.../analysis/CreateTableAsSelectStmtTest.java | 42 +++++++++++-----------
.../data/query_p0/system/test_table_options.out | 12 +++----
.../show_p0/test_show_create_table_and_views.out | 8 ++---
.../test_show_create_table_and_views_nereids.out | 8 ++---
.../test_calc_crc_fault_injection.groovy | 10 +++---
10 files changed, 69 insertions(+), 67 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
index ecce5d30920..bc25534eaaa 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
@@ -27,21 +27,21 @@ namespace doris::segment_v2 {
// {tmp_dir}/{rowset_id}_{seg_id}_{index_id}@{suffix}
std::string InvertedIndexDescriptor::get_temporary_index_path(std::string_view
tmp_dir_path,
std::string_view
rowset_id,
- int64_t seg_id,
uint64_t uuid,
+ int64_t seg_id,
int64_t index_id,
std::string_view
index_path_suffix) {
std::string suffix =
index_path_suffix.empty() ? "" : std::string {"@"} +
index_path_suffix.data();
- return fmt::format("{}/{}_{}_{}{}", tmp_dir_path, rowset_id, seg_id, uuid,
suffix);
+ return fmt::format("{}/{}_{}_{}{}", tmp_dir_path, rowset_id, seg_id,
index_id, suffix);
}
// InvertedIndexStorageFormat V1
// {prefix}_{index_id}@{suffix}.idx
std::string InvertedIndexDescriptor::get_index_path_v1(std::string_view
index_path_prefix,
- uint64_t uuid,
+ int64_t index_id,
std::string_view
index_path_suffix) {
std::string suffix =
index_path_suffix.empty() ? "" : std::string {"@"} +
index_path_suffix.data();
- return fmt::format("{}_{}{}{}", index_path_prefix, uuid, suffix,
index_suffix);
+ return fmt::format("{}_{}{}{}", index_path_prefix, index_id, suffix,
index_suffix);
}
// InvertedIndexStorageFormat V2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
index bd64bdb5eaa..01f0636f943 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
@@ -32,9 +32,10 @@ public:
static constexpr std::string_view index_suffix = ".idx";
static std::string get_temporary_index_path(std::string_view tmp_dir_path,
std::string_view rowset_id,
int64_t seg_id,
- uint64_t uuid,
std::string_view index_path_suffix);
+ int64_t index_id,
+ std::string_view
index_path_suffix);
// InvertedIndexStorageFormat V1
- static std::string get_index_path_v1(std::string_view index_path_prefix,
uint64_t uuid,
+ static std::string get_index_path_v1(std::string_view index_path_prefix,
int64_t index_id,
std::string_view index_path_suffix);
// InvertedIndexStorageFormat V2
static std::string get_index_path_v2(std::string_view index_path_prefix);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index ed38909002a..5ef014ea0ab 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -73,7 +73,7 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t
read_buffer_size) {
ReaderFileEntry* entry = nullptr;
for (int32_t i = 0; i < numIndices; ++i) {
- int64_t indexId = _stream->readInt(); // Read index ID
+ int64_t indexId = _stream->readLong(); // Read index ID
int32_t suffix_length = _stream->readInt(); // Read suffix
length
std::vector<uint8_t> suffix_data(suffix_length);
_stream->readBytes(suffix_data.data(), suffix_length);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
index 4b5f9dd54cc..b7cb9591b88 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
@@ -99,22 +99,23 @@ Status InvertedIndexFileWriter::delete_index(const
TabletIndex* index_meta) {
size_t InvertedIndexFileWriter::headerLength() {
size_t header_size = 0;
header_size +=
- sizeof(int) * 2; // Account for the size of the version number and
number of indices
+ sizeof(int32_t) * 2; // Account for the size of the version number
and number of indices
+
for (const auto& entry : _indices_dirs) {
- auto suffix = entry.first.second;
- header_size += sizeof(int); // index id
- header_size += 4; // index suffix name size
+ const auto& suffix = entry.first.second;
+ header_size += sizeof(int64_t); // index id
+ header_size += sizeof(int32_t); // index suffix name size
header_size += suffix.length(); // index suffix name
- header_size += sizeof(int); // index file count
+ header_size += sizeof(int32_t); // index file count
const auto& dir = entry.second;
std::vector<std::string> files;
dir->list(&files);
- for (auto file : files) {
- header_size += 4; // file name size
- header_size += file.length(); // file name
- header_size += 8; // file offset
- header_size += 8; // file size
+ for (const auto& file : files) {
+ header_size += sizeof(int32_t); // file name size
+ header_size += file.length(); // file name
+ header_size += sizeof(int64_t); // file offset
+ header_size += sizeof(int64_t); // file size
}
}
return header_size;
@@ -192,10 +193,10 @@ size_t InvertedIndexFileWriter::write() {
}
// sort file list by file length
std::vector<std::pair<std::string, int64_t>> sorted_files;
- for (auto file : files) {
+ for (const auto& file : files) {
sorted_files.emplace_back(file, dir->fileLength(file.c_str()));
}
- // TODO: need to optimize
+
std::sort(sorted_files.begin(), sorted_files.end(),
[](const std::pair<std::string, int64_t>& a,
const std::pair<std::string, int64_t>& b) { return
(a.second < b.second); });
@@ -203,18 +204,18 @@ size_t InvertedIndexFileWriter::write() {
int32_t file_count = sorted_files.size();
// Write the index ID and the number of files
- compound_file_output->writeInt(index_id);
- const auto* index_suffix_str = reinterpret_cast<const
uint8_t*>(index_suffix.c_str());
- compound_file_output->writeInt(index_suffix.length());
- compound_file_output->writeBytes(index_suffix_str,
index_suffix.length());
+ compound_file_output->writeLong(index_id);
+
compound_file_output->writeInt(static_cast<int32_t>(index_suffix.length()));
+ compound_file_output->writeBytes(reinterpret_cast<const
uint8_t*>(index_suffix.data()),
+ index_suffix.length());
compound_file_output->writeInt(file_count);
// Calculate the offset for each file and write the file metadata
for (const auto& file : sorted_files) {
int64_t file_length = dir->fileLength(file.first.c_str());
- const auto* file_name = reinterpret_cast<const
uint8_t*>(file.first.c_str());
- compound_file_output->writeInt(file.first.length());
- compound_file_output->writeBytes(file_name, file.first.length());
+
compound_file_output->writeInt(static_cast<int32_t>(file.first.length()));
+ compound_file_output->writeBytes(reinterpret_cast<const
uint8_t*>(file.first.data()),
+ file.first.length());
compound_file_output->writeLong(current_offset);
compound_file_output->writeLong(file_length);
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index db2f2700f80..615e034e568 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2620,7 +2620,7 @@ public class Config extends ConfigBase {
"倒排索引默认存储格式",
"Default storage format of inverted index, the default value is
V1."
})
- public static String inverted_index_storage_format = "V1";
+ public static String inverted_index_storage_format = "V2";
@ConfField(description = {
"是否开启 Proxy Protocol 支持",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index be1ddf7bc4a..f52a7aea6a2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -96,7 +96,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -122,7 +122,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" =
\"false\",\n"
@@ -144,7 +144,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" =
\"false\",\n"
@@ -184,7 +184,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -213,7 +213,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -244,7 +244,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -272,7 +272,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -296,7 +296,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -327,7 +327,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -355,7 +355,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -387,7 +387,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -416,7 +416,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -444,7 +444,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -468,7 +468,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -499,7 +499,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -529,7 +529,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -558,7 +558,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
@@ -588,7 +588,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"enable_unique_key_merge_on_write\" = \"true\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -643,7 +643,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" =
\"V1\",\n"
+ + "\"inverted_index_storage_format\" =
\"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" =
\"false\",\n"
@@ -665,7 +665,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" =
\"V1\",\n"
+ + "\"inverted_index_storage_format\" =
\"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" =
\"false\",\n"
@@ -701,7 +701,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"is_being_synced\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\",\n"
+ "\"storage_format\" = \"V2\",\n"
- + "\"inverted_index_storage_format\" = \"V1\",\n"
+ + "\"inverted_index_storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
diff --git a/regression-test/data/query_p0/system/test_table_options.out
b/regression-test/data/query_p0/system/test_table_options.out
index 8700b8741dd..8a14e8c5463 100644
--- a/regression-test/data/query_p0/system/test_table_options.out
+++ b/regression-test/data/query_p0/system/test_table_options.out
@@ -1,9 +1,9 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select --
-aggregate_table internal test_table_options_db AGG
user_id,date,city,age,sex user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time [...]
-duplicate_table internal test_table_options_db DUP
timestamp,type,error_code type HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time_se [...]
-listtable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 16 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1", [...]
-randomtable internal test_table_options_db DUP
user_id,date,timestamp RANDOM RANDOM 16 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time_seri [...]
-rangetable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 8 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1", [...]
-unique_table internal test_table_options_db UNI
user_id,username user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time_series_comp [...]
+aggregate_table internal test_table_options_db AGG
user_id,date,city,age,sex user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time [...]
+duplicate_table internal test_table_options_db DUP
timestamp,type,error_code type HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time_se [...]
+listtable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 16 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1", [...]
+randomtable internal test_table_options_db DUP
user_id,date,timestamp RANDOM RANDOM 16 1
{"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time_seri [...]
+rangetable internal test_table_options_db AGG
user_id,date,timestamp,city,age,sex user_id HASH 8 3
{"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1", [...]
+unique_table internal test_table_options_db UNI
user_id,username user_id HASH 1 1
{"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default:
1","time_series_compaction_level_threshold":"1","time_series_comp [...]
diff --git a/regression-test/data/show_p0/test_show_create_table_and_views.out
b/regression-test/data/show_p0/test_show_create_table_and_views.out
index 55b41677ac9..0bc4f97496c 100644
--- a/regression-test/data/show_p0/test_show_create_table_and_views.out
+++ b/regression-test/data/show_p0/test_show_create_table_and_views.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !show --
-show_create_table_and_views_table CREATE TABLE
`show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), ("400")),\nPAR [...]
+show_create_table_and_views_table CREATE TABLE
`show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), ("400")),\nPAR [...]
-- !select --
1 1 30
@@ -36,11 +36,11 @@ show_create_table_and_views_view CREATE VIEW
`show_create_table_and_views_view`
300 1
-- !show --
-show_create_table_and_views_table CREATE TABLE
`show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), ("400")),\nPAR [...]
+show_create_table_and_views_table CREATE TABLE
`show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), ("400")),\nPAR [...]
-- !show --
-show_create_table_and_views_like CREATE TABLE
`show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), ("400")),\nPARTI [...]
+show_create_table_and_views_like CREATE TABLE
`show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), ("400")),\nPARTI [...]
-- !show --
-show_create_table_and_views_like_with_rollup CREATE TABLE
`show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT
NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [( [...]
+show_create_table_and_views_like_with_rollup CREATE TABLE
`show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT
NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [( [...]
diff --git
a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
index 3a5728d3d9e..bbd716a7249 100644
--- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
+++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !show --
-show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300") [...]
+show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300") [...]
-- !select --
1 1 30
@@ -36,11 +36,11 @@ show_create_table_and_views_nereids_view CREATE VIEW
`show_create_table_and_view
300 1
-- !show --
-show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300") [...]
+show_create_table_and_views_nereids_table CREATE TABLE
`show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300") [...]
-- !show --
-show_create_table_and_views_nereids_like CREATE TABLE
`show_create_table_and_views_nereids_like` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), [...]
+show_create_table_and_views_nereids_like CREATE TABLE
`show_create_table_and_views_nereids_like` (\n `user_id` LARGEINT NOT NULL,\n
`good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n)
ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY
RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4
VALUES [("300"), [...]
-- !show --
-show_create_table_and_views_nereids_like_with_rollup CREATE TABLE
`show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` LARGEINT
NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT
"0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT
'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTIT [...]
+show_create_table_and_views_nereids_like_with_rollup CREATE TABLE
`show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` LARGEINT
NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT
"0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT
'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES
[("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES
[("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTIT [...]
diff --git
a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy
index e5cb5526d9f..f4971f3fe56 100644
---
a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy
@@ -73,7 +73,7 @@ suite("test_calc_crc") {
assertEquals("0", parseJson(out_0.trim()).start_version)
assertEquals("7", parseJson(out_0.trim()).end_version)
assertEquals("7", parseJson(out_0.trim()).rowset_count)
- assertEquals("18", parseJson(out_0.trim()).file_count)
+ assertEquals("12", parseJson(out_0.trim()).file_count)
try {
GetDebugPoint().enableDebugPointForAllBEs("fault_inject::BetaRowset::calc_local_file_crc")
@@ -90,7 +90,7 @@ suite("test_calc_crc") {
assertEquals("0", parseJson(out_2.trim()).start_version)
assertEquals("7", parseJson(out_2.trim()).end_version)
assertEquals("7", parseJson(out_2.trim()).rowset_count)
- assertEquals("18", parseJson(out_2.trim()).file_count)
+ assertEquals("12", parseJson(out_2.trim()).file_count)
assertTrue(parseJson(out_0.trim()).crc_value ==
parseJson(out_2.trim()).crc_value)
def (code_3, out_3, err_3) = calc_file_crc_on_tablet_with_end(ip, port,
tablet_id, 7)
@@ -99,7 +99,7 @@ suite("test_calc_crc") {
assertEquals("0", parseJson(out_3.trim()).start_version)
assertEquals("7", parseJson(out_3.trim()).end_version)
assertEquals("7", parseJson(out_3.trim()).rowset_count)
- assertEquals("18", parseJson(out_3.trim()).file_count)
+ assertEquals("12", parseJson(out_3.trim()).file_count)
assertTrue(parseJson(out_2.trim()).crc_value ==
parseJson(out_3.trim()).crc_value)
def (code_4, out_4, err_4) = calc_file_crc_on_tablet_with_start_end(ip,
port, tablet_id, 3, 6)
@@ -109,7 +109,7 @@ suite("test_calc_crc") {
assertEquals("3", parseJson(out_4.trim()).start_version)
assertEquals("6", parseJson(out_4.trim()).end_version)
assertEquals("4", parseJson(out_4.trim()).rowset_count)
- assertEquals("12", parseJson(out_4.trim()).file_count)
+ assertEquals("8", parseJson(out_4.trim()).file_count)
def (code_5, out_5, err_5) = calc_file_crc_on_tablet_with_start_end(ip,
port, tablet_id, 5, 9)
logger.info("Run calc_file_crc_on_tablet: code=" + code_5 + ", out=" +
out_5 + ", err=" + err_5)
@@ -118,7 +118,7 @@ suite("test_calc_crc") {
assertEquals("5", parseJson(out_5.trim()).start_version)
assertEquals("7", parseJson(out_5.trim()).end_version)
assertEquals("3", parseJson(out_5.trim()).rowset_count)
- assertEquals("9", parseJson(out_5.trim()).file_count)
+ assertEquals("6", parseJson(out_5.trim()).file_count)
def (code_6, out_6, err_6) = calc_file_crc_on_tablet(ip, port, 123)
logger.info("Run calc_file_crc_on_tablet: code=" + code_6 + ", out=" +
out_6 + ", err=" + err_6)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
