This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 82c3f379b1 [Load](Sink) remove validate the column data when data is
NULL (#13934)
82c3f379b1 is described below
commit 82c3f379b126d9ecf186eddeddbc102421e065f3
Author: HappenLee <[email protected]>
AuthorDate: Thu Nov 3 14:24:58 2022 +0800
[Load](Sink) remove validate the column data when data is NULL (#13934)
---
be/src/vec/sink/vtablet_sink.cpp | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp
index 8989dd544e..7aca4fed5f 100644
--- a/be/src/vec/sink/vtablet_sink.cpp
+++ b/be/src/vec/sink/vtablet_sink.cpp
@@ -197,6 +197,10 @@ Status VOlapTableSink::_validate_data(RuntimeState* state,
vectorized::Block* bl
auto column_ptr =
vectorized::check_and_get_column<vectorized::ColumnNullable>(*column);
auto& real_column_ptr =
column_ptr == nullptr ? column :
(column_ptr->get_nested_column_ptr());
+ auto null_map = column_ptr == nullptr ? nullptr :
column_ptr->get_null_map_data().data();
+ auto need_to_validate = [&null_map, &filter_bitmap](int j) {
+ return !filter_bitmap->Get(j) && (null_map == nullptr ||
null_map[j] == 0);
+ };
switch (desc->type().type) {
case TYPE_CHAR:
@@ -210,7 +214,7 @@ Status VOlapTableSink::_validate_data(RuntimeState* state,
vectorized::Block* bl
limit = std::min(config::string_type_length_soft_limit_bytes,
desc->type().len);
}
for (int j = 0; j < num_rows; ++j) {
- if (!filter_bitmap->Get(j)) {
+ if (need_to_validate(j)) {
auto str_val = column_string->get_data_at(j);
bool invalid = str_val.size > limit;
if (invalid) {
@@ -244,7 +248,7 @@ Status VOlapTableSink::_validate_data(RuntimeState* state,
vectorized::Block* bl
real_column_ptr.get()));
for (int j = 0; j < num_rows; ++j) {
- if (!filter_bitmap->Get(j)) {
+ if (need_to_validate(j)) {
auto dec_val = binary_cast<vectorized::Int128,
DecimalV2Value>(
column_decimal->get_data()[j]);
error_msg.clear();
@@ -287,8 +291,7 @@ Status VOlapTableSink::_validate_data(RuntimeState* state,
vectorized::Block* bl
// 1. column is nullable but the desc is not nullable
// 2. desc->type is BITMAP
if ((!desc->is_nullable() || desc->type() == TYPE_OBJECT) &&
column_ptr) {
- const auto& null_map = column_ptr->get_null_map_data();
- for (int j = 0; j < null_map.size(); ++j) {
+ for (int j = 0; j < num_rows; ++j) {
if (null_map[j] && !filter_bitmap->Get(j)) {
error_msg.clear();
fmt::format_to(error_msg, "null value for not null column,
column={}; ",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
