This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 35a282fd61 [BugFix] Column datas doesn't match nullmap when
vectorization load (#10684)
35a282fd61 is described below
commit 35a282fd6112b36e858d6739e73aa30a1e9b1d64
Author: Lightman <[email protected]>
AuthorDate: Fri Jul 8 17:39:44 2022 +0800
[BugFix] Column datas doesn't match nullmap when vectorization load (#10684)
* block column doesn't match nullmap
* remove _nullmap+_row_pos in convertor_to_olap
---
be/src/vec/olap/olap_data_convertor.cpp | 46 ++++++++++++++++-----------------
be/src/vec/olap/olap_data_convertor.h | 2 +-
2 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/be/src/vec/olap/olap_data_convertor.cpp
b/be/src/vec/olap/olap_data_convertor.cpp
index 72a55e545b..03a1a208cd 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -138,6 +138,7 @@ void
OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(
auto nullable_column =
assert_cast<const
vectorized::ColumnNullable*>(_typed_column.column.get());
_nullmap = nullable_column->get_null_map_data().data();
+ _nullmap += row_pos;
}
}
@@ -194,7 +195,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
size_t total_size = 0;
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (bitmap_value_cur != bitmap_value_end) {
if (!*nullmap_cur) {
total_size += bitmap_value_cur->getSizeInBytes();
@@ -215,7 +216,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
char* raw_data = _raw_data.data();
Slice* slice = _slice.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (bitmap_value_cur != bitmap_value_end) {
if (!*nullmap_cur) {
slice_size = bitmap_value_cur->getSizeInBytes();
@@ -233,7 +234,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
++nullmap_cur;
++bitmap_value_cur;
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice ==
_slice.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && slice ==
_slice.get_end_ptr());
} else {
while (bitmap_value_cur != bitmap_value_end) {
slice_size = bitmap_value_cur->getSizeInBytes();
@@ -254,8 +255,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
assert(_typed_column.column);
const vectorized::ColumnHLL* column_hll = nullptr;
- const UInt8* nullmap = get_nullmap();
- if (nullmap) {
+ if (_nullmap) {
auto nullable_column =
assert_cast<const
vectorized::ColumnNullable*>(_typed_column.column.get());
column_hll = assert_cast<const vectorized::ColumnHLL*>(
@@ -270,8 +270,8 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
HyperLogLog* hll_value_end = hll_value_cur + _num_rows;
size_t total_size = 0;
- if (nullmap) {
- const UInt8* nullmap_cur = nullmap + _row_pos;
+ if (_nullmap) {
+ const UInt8* nullmap_cur = _nullmap;
while (hll_value_cur != hll_value_end) {
if (!*nullmap_cur) {
total_size += hll_value_cur->max_serialized_size();
@@ -292,8 +292,8 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
Slice* slice = _slice.data();
hll_value_cur = hll_value;
- if (nullmap) {
- const UInt8* nullmap_cur = nullmap + _row_pos;
+ if (_nullmap) {
+ const UInt8* nullmap_cur = _nullmap;
while (hll_value_cur != hll_value_end) {
if (!*nullmap_cur) {
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
@@ -310,7 +310,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
++nullmap_cur;
++hll_value_cur;
}
- assert(nullmap_cur == nullmap + _row_pos + _num_rows && slice ==
_slice.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && slice ==
_slice.get_end_ptr());
} else {
while (hll_value_cur != hll_value_end) {
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
@@ -372,7 +372,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorChar::convert_to_olap() {
}
for (size_t i = 0; i < _num_rows; i++) {
- if (!_nullmap || !_nullmap[i + _row_pos]) {
+ if (!_nullmap || !_nullmap[i]) {
_slice[i] = column_string->get_data_at(i + _row_pos).to_slice();
DCHECK(_slice[i].size == _length)
<< "char type data length not equal to schema, schema=" <<
_length
@@ -430,7 +430,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
Slice* slice = _slice.data();
size_t string_offset = *(offset_cur - 1);
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (offset_cur != offset_end) {
if (!*nullmap_cur) {
slice->data = const_cast<char*>(char_data + string_offset);
@@ -451,7 +451,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
++slice;
++offset_cur;
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice ==
_slice.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && slice ==
_slice.get_end_ptr());
} else {
while (offset_cur != offset_end) {
slice->data = const_cast<char*>(char_data + string_offset);
@@ -503,7 +503,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
const DateV2Value* datetime_end = datetime_cur + _num_rows;
uint24_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_date();
@@ -514,8 +514,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
++datetime_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
- value == _values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && value ==
_values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
@@ -544,7 +543,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint24_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_date();
@@ -555,8 +554,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
++datetime_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
- value == _values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && value ==
_values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
@@ -589,7 +587,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint64_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_datetime();
@@ -600,7 +598,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
++datetime_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value ==
_values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && value ==
_values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_datetime();
@@ -632,7 +630,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
const DecimalV2Value* decimal_end = decimal_cur + _num_rows;
decimal12_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (decimal_cur != decimal_end) {
if (!*nullmap_cur) {
value->integer = decimal_cur->int_value();
@@ -644,7 +642,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
++decimal_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value ==
_values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _num_rows && value ==
_values.get_end_ptr());
} else {
while (decimal_cur != decimal_end) {
value->integer = decimal_cur->int_value();
@@ -705,7 +703,7 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap(
for (size_t i = 0; i < _num_rows; ++i, ++collection_value) {
int64_t cur_pos = _row_pos + i;
int64_t prev_pos = cur_pos - 1;
- if (_nullmap && _nullmap[cur_pos]) {
+ if (_nullmap && _nullmap[cur_pos - _row_pos]) {
continue;
}
auto offset = offsets[prev_pos];
diff --git a/be/src/vec/olap/olap_data_convertor.h
b/be/src/vec/olap/olap_data_convertor.h
index 97bf3a7532..4fad5b6f59 100644
--- a/be/src/vec/olap/olap_data_convertor.h
+++ b/be/src/vec/olap/olap_data_convertor.h
@@ -298,7 +298,7 @@ private:
const VecDateTimeValue* datetime_end = datetime_cur +
_num_rows;
uint32_t* value = const_cast<uint32_t*>(values_);
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
+ const UInt8* nullmap_cur = _nullmap;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_date_v2();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
