This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 309152d92fc branch-4.1: [opt](serde)Build Arrow serde error context
lazily. #64660 (#64712)
309152d92fc is described below
commit 309152d92fcc2a0454f51b0a8e31efbe3680701f
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jun 23 19:20:42 2026 +0800
branch-4.1: [opt](serde)Build Arrow serde error context lazily. #64660
(#64712)
Cherry-picked from #64660
Co-authored-by: daidai <[email protected]>
---
.../core/data_type_serde/data_type_array_serde.cpp | 6 ++---
.../data_type_serde/data_type_bitmap_serde.cpp | 5 ++--
.../data_type_date_or_datetime_serde.cpp | 5 ++--
.../data_type_serde/data_type_datetimev2_serde.cpp | 8 +++---
.../data_type_serde/data_type_datev2_serde.cpp | 5 ++--
.../data_type_serde/data_type_decimal_serde.cpp | 30 ++++++++--------------
.../core/data_type_serde/data_type_hll_serde.cpp | 5 ++--
.../core/data_type_serde/data_type_ipv4_serde.cpp | 2 +-
.../core/data_type_serde/data_type_ipv6_serde.cpp | 5 ++--
.../core/data_type_serde/data_type_jsonb_serde.cpp | 5 ++--
.../core/data_type_serde/data_type_map_serde.cpp | 9 +++----
.../data_type_serde/data_type_number_serde.cpp | 20 +++++++--------
.../data_type_quantilestate_serde.h | 5 ++--
be/src/core/data_type_serde/data_type_serde.h | 7 ++---
.../data_type_serde/data_type_string_serde.cpp | 5 ++--
.../data_type_serde/data_type_struct_serde.cpp | 6 ++---
.../data_type_timestamptz_serde.cpp | 8 +++---
.../data_type_serde/data_type_varbinary_serde.cpp | 5 ++--
.../data_type_serde/data_type_variant_serde.cpp | 5 ++--
19 files changed, 60 insertions(+), 86 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp
b/be/src/core/data_type_serde/data_type_array_serde.cpp
index c7b139083df..dbabe2053f1 100644
--- a/be/src/core/data_type_serde/data_type_array_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_array_serde.cpp
@@ -305,12 +305,10 @@ Status DataTypeArraySerDe::write_column_to_arrow(const
IColumn& column, const Nu
auto* nested_builder = builder.value_builder();
for (size_t array_idx = start; array_idx < end; ++array_idx) {
if (null_map && (*null_map)[array_idx]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column,
*array_builder));
RETURN_IF_ERROR(nested_serde->write_column_to_arrow(nested_data,
nullptr, nested_builder,
offsets[array_idx
- 1],
offsets[array_idx], ctz));
diff --git a/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
b/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
index fbd1b470398..b68fcc772e0 100644
--- a/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
@@ -128,15 +128,14 @@ Status DataTypeBitMapSerDe::write_column_to_arrow(const
IColumn& column, const N
auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else {
auto& bitmap_value = col.get_element(string_i);
std::string memory_buffer(bitmap_value.getSizeInBytes(), '0');
bitmap_value.write_to(memory_buffer.data());
RETURN_IF_ERROR(checkArrowStatus(
builder.Append(memory_buffer.data(),
static_cast<int>(memory_buffer.size())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
index 40561b9d886..c7c9eb19630 100644
--- a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
@@ -157,11 +157,10 @@ Status DataTypeDateSerDe<T>::write_column_to_arrow(const
IColumn& column, const
const auto* time_val = (const VecDateTimeValue*)(&col_data[i]);
size_t len = time_val->to_buffer(buf);
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column, *array_builder));
} else {
RETURN_IF_ERROR(checkArrowStatus(string_builder.Append(buf,
cast_set<int32_t>(len)),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
index c368ad17662..7a0dfd163c7 100644
--- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
@@ -373,8 +373,8 @@ Status DataTypeDateTimeV2SerDe::write_column_to_arrow(const
IColumn& column,
const cctz::time_zone& real_ctz = timezone.empty() ? cctz::utc_time_zone()
: ctz;
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.AppendNull(), column,
*array_builder));
} else {
int64_t timestamp = 0;
DateV2Value<DateTimeV2ValueType> datetime_val = col_data[i];
@@ -387,8 +387,8 @@ Status DataTypeDateTimeV2SerDe::write_column_to_arrow(const
IColumn& column,
uint32_t millisecond = datetime_val.microsecond() / 1000;
timestamp = (timestamp * 1000) + millisecond;
}
-
RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.Append(timestamp),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.Append(timestamp),
column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp
b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
index 34d69fc1dd1..17c32f04ebe 100644
--- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
@@ -97,12 +97,11 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const
IColumn& column, const N
for (size_t i = start; i < end; ++i) {
auto daynr = col_data[i].daynr() - date_threshold;
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(),
column, *array_builder));
} else {
RETURN_IF_ERROR(
checkArrowStatus(date32_builder.Append(cast_set<int,
int64_t, false>(daynr)),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
index aace06c0152..9e2864ed854 100644
--- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
@@ -239,8 +239,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::make_shared<arrow::Decimal128Type>(27, 9);
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
const auto& data_ref = col.get_data_at(i);
@@ -248,8 +247,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
int64_t high = (p_value->value) >> 64;
uint64_t low = cast_set<uint64_t>((p_value->value) &
0xFFFFFFFFFFFFFFFF);
arrow::Decimal128 value(high, low);
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL128I) {
auto& builder =
reinterpret_cast<arrow::Decimal128Builder&>(*array_builder);
@@ -257,8 +255,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::make_shared<arrow::Decimal128Type>(38, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
const auto& data_ref = col.get_data_at(i);
@@ -266,8 +263,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
int64_t high = (p_value->value) >> 64;
uint64_t low = cast_set<uint64_t>((p_value->value) &
0xFFFFFFFFFFFFFFFF);
arrow::Decimal128 value(high, low);
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL32) {
auto& builder =
reinterpret_cast<arrow::Decimal128Builder&>(*array_builder);
@@ -275,14 +271,12 @@ Status
DataTypeDecimalSerDe<T>::write_column_to_arrow(const IColumn& column,
std::make_shared<arrow::Decimal128Type>(8, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
Int128 p_value = col.get_element(i).value;
arrow::Decimal128 value(reinterpret_cast<const
uint8_t*>(&p_value));
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL64) {
auto& builder =
reinterpret_cast<arrow::Decimal128Builder&>(*array_builder);
@@ -290,14 +284,12 @@ Status
DataTypeDecimalSerDe<T>::write_column_to_arrow(const IColumn& column,
std::make_shared<arrow::Decimal128Type>(18, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
Int128 p_value = col.get_element(i).value;
arrow::Decimal128 value(reinterpret_cast<const
uint8_t*>(&p_value));
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL256) {
auto& builder =
reinterpret_cast<arrow::Decimal256Builder&>(*array_builder);
@@ -305,8 +297,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::make_shared<arrow::Decimal256Type>(76, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
auto p_value = wide::Int256(col.get_element(i));
@@ -318,8 +309,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::array<uint64_t, 4> word_array = {a0, a1, a2, a3};
arrow::Decimal256 value(arrow::Decimal256::LittleEndianArray,
word_array);
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else {
return Status::InvalidArgument("write_column_to_arrow with type " +
column.get_name());
diff --git a/be/src/core/data_type_serde/data_type_hll_serde.cpp
b/be/src/core/data_type_serde/data_type_hll_serde.cpp
index a90a5f44d70..568ecee7db4 100644
--- a/be/src/core/data_type_serde/data_type_hll_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_hll_serde.cpp
@@ -132,15 +132,14 @@ Status DataTypeHLLSerDe::write_column_to_arrow(const
IColumn& column, const Null
auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else {
auto& hll_value =
const_cast<HyperLogLog&>(col.get_element(string_i));
std::string memory_buffer(hll_value.max_serialized_size(), '0');
hll_value.serialize((uint8_t*)memory_buffer.data());
RETURN_IF_ERROR(checkArrowStatus(
builder.Append(memory_buffer.data(),
static_cast<int>(memory_buffer.size())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
b/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
index 1b3978f929a..fdb091fe835 100644
--- a/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
@@ -107,7 +107,7 @@ Status DataTypeIPv4SerDe::write_column_to_arrow(const
IColumn& column, const Nul
int32_builder.AppendValues(reinterpret_cast<const
Int32*>(col_data.data()) + start,
end - start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
b/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
index e860662e404..febf802d0e0 100644
--- a/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
@@ -128,14 +128,13 @@ Status DataTypeIPv6SerDe::write_column_to_arrow(const
IColumn& column, const Nul
auto& string_builder = assert_cast<arrow::StringBuilder&>(*array_builder);
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column, *array_builder));
} else {
std::string ipv6_str = IPv6Value::to_string(col_data[i]);
RETURN_IF_ERROR(checkArrowStatus(
string_builder.Append(ipv6_str.c_str(),
cast_set<int, size_t,
false>(ipv6_str.size())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
index 4b0c5b2e2c3..90660dac372 100644
--- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
@@ -104,8 +104,7 @@ Status DataTypeJsonbSerDe::write_column_to_arrow(const
IColumn& column, const Nu
auto& builder = assert_cast<arrow::StringBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
std::string_view string_ref =
string_column.get_data_at(string_i).to_string_view();
@@ -114,7 +113,7 @@ Status DataTypeJsonbSerDe::write_column_to_arrow(const
IColumn& column, const Nu
RETURN_IF_ERROR(
checkArrowStatus(builder.Append(json_string.data(),
cast_set<int, size_t,
false>(json_string.size())),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp
b/be/src/core/data_type_serde/data_type_map_serde.cpp
index 44fbde623e7..5f00a887ff0 100644
--- a/be/src/core/data_type_serde/data_type_map_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_map_serde.cpp
@@ -349,8 +349,7 @@ Status DataTypeMapSerDe::write_column_to_arrow(const
IColumn& column, const Null
for (size_t r = start; r < end; ++r) {
if ((null_map && (*null_map)[r])) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else if (simd::contain_one(keys_nullmap_data + offsets[r - 1],
offsets[r] - offsets[r - 1])) {
// arrow do not support key is null, so we ignore the null
key-value
@@ -364,8 +363,7 @@ Status DataTypeMapSerDe::write_column_to_arrow(const
IColumn& column, const Null
key_mutable_data->insert_from(nested_keys_column, i);
value_mutable_data->insert_from(nested_values_column, i);
}
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column,
*array_builder));
RETURN_IF_ERROR(key_serde->write_column_to_arrow(
*key_mutable_data, nullptr, key_builder, 0,
key_mutable_data->size(), ctz));
@@ -373,8 +371,7 @@ Status DataTypeMapSerDe::write_column_to_arrow(const
IColumn& column, const Null
value_builder,
0,
value_mutable_data->size(), ctz));
} else {
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column,
*array_builder));
RETURN_IF_ERROR(key_serde->write_column_to_arrow(
nested_keys_column, nullptr, key_builder, offsets[r - 1],
offsets[r], ctz));
RETURN_IF_ERROR(value_serde->write_column_to_arrow(
diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp
b/be/src/core/data_type_serde/data_type_number_serde.cpp
index 610a24573ce..f82b33769c5 100644
--- a/be/src/core/data_type_serde/data_type_number_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_number_serde.cpp
@@ -101,8 +101,8 @@ Status DataTypeNumberSerDe<T>::write_column_to_arrow(const
IColumn& column, cons
auto* null_builder = dynamic_cast<arrow::NullBuilder*>(array_builder);
if (null_builder) {
for (size_t i = start; i < end; ++i) {
- RETURN_IF_ERROR(checkArrowStatus(null_builder->AppendNull(),
column.get_name(),
-
null_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(null_builder->AppendNull(), column,
*null_builder));
}
} else {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
@@ -110,7 +110,7 @@ Status DataTypeNumberSerDe<T>::write_column_to_arrow(const
IColumn& column, cons
builder.AppendValues(reinterpret_cast<const
uint8_t*>(col_data.data() + start),
end - start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
} else if constexpr (T == TYPE_LARGEINT) {
@@ -119,13 +119,13 @@ Status
DataTypeNumberSerDe<T>::write_column_to_arrow(const IColumn& column, cons
auto& data_value = col_data[i];
std::string value_str = fmt::format("{}", data_value);
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(string_builder.AppendNull(), column,
*array_builder));
} else {
RETURN_IF_ERROR(checkArrowStatus(
string_builder.Append(value_str.data(),
cast_set<int, size_t,
false>(value_str.length())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
} else if constexpr (T == TYPE_IPV6) {
@@ -134,25 +134,25 @@ Status
DataTypeNumberSerDe<T>::write_column_to_arrow(const IColumn& column, cons
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues((int64_t*)col_data.data() + start, end -
start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
} else if constexpr (T == TYPE_DATEV2) {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues((uint32_t*)col_data.data() + start, end -
start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
} else if constexpr (T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues((uint64_t*)col_data.data() + start, end -
start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
} else {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues(col_data.data() + start, end - start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_quantilestate_serde.h
b/be/src/core/data_type_serde/data_type_quantilestate_serde.h
index 730a7a64a78..bca1b4b4687 100644
--- a/be/src/core/data_type_serde/data_type_quantilestate_serde.h
+++ b/be/src/core/data_type_serde/data_type_quantilestate_serde.h
@@ -105,8 +105,7 @@ public:
auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else {
auto& quantile_state_value =
const_cast<QuantileState&>(col.get_element(string_i));
std::string
memory_buffer(quantile_state_value.get_serialized_size(), '0');
@@ -114,7 +113,7 @@ public:
RETURN_IF_ERROR(
checkArrowStatus(builder.Append(memory_buffer.data(),
static_cast<int>(memory_buffer.size())),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_serde.h
b/be/src/core/data_type_serde/data_type_serde.h
index b9a5cbe76bd..4876f2e0b27 100644
--- a/be/src/core/data_type_serde/data_type_serde.h
+++ b/be/src/core/data_type_serde/data_type_serde.h
@@ -546,11 +546,12 @@ inline static NullMap revert_null_map(const NullMap*
null_bytemap, size_t start,
return res;
}
-inline Status checkArrowStatus(const arrow::Status& status, const std::string&
column,
- const std::string& format_name) {
+template <typename ColumnType, typename BuilderType>
+inline Status checkArrowStatus(const arrow::Status& status, const ColumnType&
column,
+ const BuilderType& builder) {
if (!status.ok()) {
return Status::FatalError("arrow serde with arrow: {} with column : {}
with error msg: {}",
- format_name, column, status.ToString());
+ builder.type()->name(), column.get_name(),
status.ToString());
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp
b/be/src/core/data_type_serde/data_type_string_serde.cpp
index 7d88fcdf809..e30fbea7c30 100644
--- a/be/src/core/data_type_serde/data_type_string_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_string_serde.cpp
@@ -225,14 +225,13 @@ Status
DataTypeStringSerDeBase<ColumnType>::write_column_to_arrow_impl(const ICo
const auto& string_column = assert_cast<const ColumnType&>(column);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
builder));
continue;
}
auto string_ref = string_column.get_data_at(string_i);
RETURN_IF_ERROR(checkArrowStatus(
builder.Append(string_ref.data, cast_set<int, size_t,
false>(string_ref.size)),
- column.get_name(), builder.type()->name()));
+ column, builder));
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp
b/be/src/core/data_type_serde/data_type_struct_serde.cpp
index f64149734f7..ee0643bc6aa 100644
--- a/be/src/core/data_type_serde/data_type_struct_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp
@@ -402,12 +402,10 @@ Status DataTypeStructSerDe::write_column_to_arrow(const
IColumn& column, const N
const auto& struct_column = assert_cast<const ColumnStruct&>(column);
for (auto r = start; r < end; ++r) {
if (null_map != nullptr && (*null_map)[r]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
struct_column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
struct_column, builder));
continue;
}
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(),
struct_column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), struct_column,
builder));
for (auto ei = 0; ei < struct_column.tuple_size(); ++ei) {
auto* elem_builder = builder.field_builder(ei);
RETURN_IF_ERROR(elem_serdes_ptrs[ei]->write_column_to_arrow(
diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
index c22c65dbd5a..e8c26f6db68 100644
--- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
@@ -201,8 +201,8 @@ Status
DataTypeTimeStampTzSerDe::write_column_to_arrow(const IColumn& column,
static const auto& UTC = cctz::utc_time_zone();
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.AppendNull(), column,
*array_builder));
} else {
int64_t timestamp = 0;
const auto& tz = col_data[i];
@@ -215,8 +215,8 @@ Status
DataTypeTimeStampTzSerDe::write_column_to_arrow(const IColumn& column,
uint32_t millisecond = tz.microsecond() / 1000;
timestamp = (timestamp * 1000) + millisecond;
}
-
RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.Append(timestamp),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.Append(timestamp),
column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
index 76ac43d535c..2b1069d3bbb 100644
--- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
@@ -58,13 +58,12 @@ Status DataTypeVarbinarySerDe::write_column_to_arrow(const
IColumn& column, cons
const auto& varbinary_column_data = assert_cast<const
ColumnVarbinary&>(column).get_data();
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
builder));
continue;
}
const auto& string_view = varbinary_column_data[i];
RETURN_IF_ERROR(checkArrowStatus(builder.Append(string_view.data(),
string_view.size()),
- column.get_name(),
builder.type()->name()));
+ column, builder));
}
return Status::OK();
};
diff --git a/be/src/core/data_type_serde/data_type_variant_serde.cpp
b/be/src/core/data_type_serde/data_type_variant_serde.cpp
index efd35971dcb..3bd9be5797f 100644
--- a/be/src/core/data_type_serde/data_type_variant_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_variant_serde.cpp
@@ -49,8 +49,7 @@ Status write_variant_column_to_arrow_impl(const IColumn&
column, const ColumnVar
options.timezone = &ctz;
for (int64_t i = start; i < end; ++i) {
if (null_map && (*null_map)[cast_set<size_t>(i)]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
builder));
continue;
}
@@ -59,7 +58,7 @@ Status write_variant_column_to_arrow_impl(const IColumn&
column, const ColumnVar
const auto serialized_size =
cast_set<typename
BuilderType::offset_type>(serialized_value.size());
RETURN_IF_ERROR(checkArrowStatus(builder.Append(serialized_value.data(),
serialized_size),
- column.get_name(),
builder.type()->name()));
+ column, builder));
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
