This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f294d33332 [bugfix](index) index page should not be bitshuffle decoded
(#12231)
f294d33332 is described below
commit f294d33332f59ceaa54eaf39e7abbf3896d482c0
Author: TengJianPing <[email protected]>
AuthorDate: Thu Sep 1 11:56:44 2022 +0800
[bugfix](index) index page should not be bitshuffle decoded (#12231)
* [bugfix](index) index page should not be bitshuffle decoded
* minor change
---
.../rowset/segment_v2/indexed_column_reader.cpp | 9 ++--
.../olap/rowset/segment_v2/indexed_column_reader.h | 2 +-
be/src/olap/rowset/segment_v2/page_io.cpp | 2 +-
be/src/olap/rowset/segment_v2/page_io.h | 3 ++
.../test_decimal_bitmap_index_multi_page.out | 16 ++++++
regression-test/suites/index_p0/load.groovy | 58 ++++++++++++++++++++++
.../test_decimal_bitmap_index_multi_page.groovy | 25 ++++++++++
7 files changed, 109 insertions(+), 6 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 9437b218dc..02f27aa6e6 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -72,7 +72,7 @@ Status IndexedColumnReader::load_index_page(const
PagePointerPB& pp, PageHandle*
std::unique_ptr<BlockCompressionCodec> local_compress_codec;
RETURN_IF_ERROR(get_block_compression_codec(_meta.compression(),
local_compress_codec));
RETURN_IF_ERROR(read_page(PagePointer(pp), handle, &body, &footer,
INDEX_PAGE,
- local_compress_codec.get()));
+ local_compress_codec.get(), false));
RETURN_IF_ERROR(reader->parse(body, footer.index_page_footer()));
_mem_size += body.get_size();
return Status::OK();
@@ -80,7 +80,7 @@ Status IndexedColumnReader::load_index_page(const
PagePointerPB& pp, PageHandle*
Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle*
handle, Slice* body,
PageFooterPB* footer, PageTypePB type,
- BlockCompressionCodec* codec) const {
+ BlockCompressionCodec* codec, bool
pre_decode) const {
PageReadOptions opts;
opts.file_reader = _file_reader.get();
opts.page_pointer = pp;
@@ -91,6 +91,7 @@ Status IndexedColumnReader::read_page(const PagePointer& pp,
PageHandle* handle,
opts.kept_in_memory = _kept_in_memory;
opts.type = type;
opts.encoding_info = _encoding_info;
+ opts.pre_decode = pre_decode;
return PageIO::read_and_decompress_page(opts, handle, body, footer);
}
@@ -106,8 +107,8 @@ Status IndexedColumnIterator::_read_data_page(const
PagePointer& pp) {
PageHandle handle;
Slice body;
PageFooterPB footer;
- RETURN_IF_ERROR(
- _reader->read_page(pp, &handle, &body, &footer, DATA_PAGE,
_compress_codec.get()));
+ RETURN_IF_ERROR(_reader->read_page(pp, &handle, &body, &footer, DATA_PAGE,
+ _compress_codec.get(), true));
// parse data page
// note that page_index is not used in IndexedColumnIterator, so we pass 0
PageDecoderOptions opts;
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index e70b2aae5c..15b4c175d5 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -54,7 +54,7 @@ public:
// read a page specified by `pp' from `file' into `handle'
Status read_page(const PagePointer& pp, PageHandle* handle, Slice* body,
PageFooterPB* footer,
- PageTypePB type, BlockCompressionCodec* codec) const;
+ PageTypePB type, BlockCompressionCodec* codec, bool
pre_decode) const;
int64_t num_values() const { return _num_values; }
const EncodingInfo* encoding_info() const { return _encoding_info; }
diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp
b/be/src/olap/rowset/segment_v2/page_io.cpp
index d0d8db629a..4388feb6dc 100644
--- a/be/src/olap/rowset/segment_v2/page_io.cpp
+++ b/be/src/olap/rowset/segment_v2/page_io.cpp
@@ -197,7 +197,7 @@ Status PageIO::read_and_decompress_page(const
PageReadOptions& opts, PageHandle*
opts.stats->uncompressed_bytes_read += body_size;
}
- if (opts.encoding_info) {
+ if (opts.pre_decode && opts.encoding_info) {
auto* pre_decoder = opts.encoding_info->get_data_page_pre_decoder();
if (pre_decoder) {
RETURN_IF_ERROR(pre_decoder->decode(
diff --git a/be/src/olap/rowset/segment_v2/page_io.h
b/be/src/olap/rowset/segment_v2/page_io.h
index 7b8bee7358..86cff00f89 100644
--- a/be/src/olap/rowset/segment_v2/page_io.h
+++ b/be/src/olap/rowset/segment_v2/page_io.h
@@ -67,6 +67,9 @@ struct PageReadOptions {
const EncodingInfo* encoding_info = nullptr;
+ // index_page should not be pre-decoded
+ bool pre_decode = true;
+
void sanity_check() const {
CHECK_NOTNULL(file_reader);
CHECK_NOTNULL(stats);
diff --git
a/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
b/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
new file mode 100644
index 0000000000..538cbb9404
--- /dev/null
+++ b/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+a DECIMAL(12,6) No true \N
+
+-- !sql --
+default_cluster:regression_test_index_p0.test_decimal_bitmap_index_multi_page
bitmap_index_multi_page a
BITMAP
+
+-- !sql --
+0.000001
+1.000001
+2.000001
+
+-- !sql --
+131072.000001
+131071.000001
+131070.000001
diff --git a/regression-test/suites/index_p0/load.groovy
b/regression-test/suites/index_p0/load.groovy
new file mode 100644
index 0000000000..0e9d856b56
--- /dev/null
+++ b/regression-test/suites/index_p0/load.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_bitmap_index_load") {
+ def tbName = "test_decimal_bitmap_index_multi_page"
+
+ sql """
+ drop TABLE if exists `${tbName}` force;
+ """
+ sql """
+ CREATE TABLE `${tbName}` (
+ `a` decimal(12, 6) NOT NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(`a`)
+ DISTRIBUTED BY HASH(`a`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1");
+ """
+ sql """
+ create index bitmap_index_multi_page on ${tbName}(a) using bitmap;
+ """
+
+ streamLoad {
+ table "${tbName}"
+ set 'column_separator', '|'
+ set 'columns', 'a,temp'
+ file """${context.sf1DataPath}/regression/bitmap_index_test.csv"""
+ time 10000 // limit inflight 10s
+
+ // if declared a check callback, the default check condition will
ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+}
+
diff --git
a/regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy
b/regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy
new file mode 100644
index 0000000000..6607c06ad4
--- /dev/null
+++
b/regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_decimal_bitmap_index_multi_page") {
+ def tbName = "test_decimal_bitmap_index_multi_page"
+
+ qt_sql "desc ${tbName};"
+ qt_sql "SHOW INDEX FROM ${tbName};"
+ qt_sql "select * from ${tbName} order by a ASC limit 3;"
+ qt_sql "select * from ${tbName} order by a DESC limit 3;"
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
