This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new a4eddd18 fix: respect parent validity in Arrow nullability checks
(#340)
a4eddd18 is described below
commit a4eddd180ed8a8a8b29b708e5032419121d652a4
Author: lxy <[email protected]>
AuthorDate: Thu Sep 17 11:36:37 2026 +0800
fix: respect parent validity in Arrow nullability checks (#340)
---
src/paimon/common/utils/arrow/arrow_utils.cpp | 188 ++++++++++++++++-----
src/paimon/common/utils/arrow/arrow_utils.h | 6 +-
src/paimon/common/utils/arrow/arrow_utils_test.cpp | 183 +++++++++++++++++++-
3 files changed, 334 insertions(+), 43 deletions(-)
diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp
b/src/paimon/common/utils/arrow/arrow_utils.cpp
index 5a022240..77e32168 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils.cpp
@@ -39,7 +39,6 @@
#include "arrow/util/compression.h"
#include "fmt/format.h"
#include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/arrow/vector_utils.h"
#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/string_utils.h"
#include "paimon/core/casting/casting_utils.h"
@@ -97,6 +96,153 @@ bool HasUndeclaredDictionaryChild(const
std::shared_ptr<arrow::DataType>& type,
return false;
}
+// Positions in the current array which remain visible through every nullable
ancestor.
+struct VisibleRange {
+ int64_t offset;
+ int64_t length;
+};
+
+using VisibleRanges = std::vector<VisibleRange>;
+
+void AppendVisibleRange(int64_t offset, int64_t length, VisibleRanges* ranges)
{
+ if (length == 0) {
+ return;
+ }
+ if (!ranges->empty()) {
+ VisibleRange& last = ranges->back();
+ if (last.offset + last.length == offset) {
+ last.length += length;
+ return;
+ }
+ }
+ ranges->push_back({offset, length});
+}
+
+Status NullabilityMismatch(const arrow::Field& field) {
+ return Status::Invalid(fmt::format(
+ "CheckNullabilityMatch failed, field {} not nullable while data have
null value",
+ field.name()));
+}
+
+Result<VisibleRanges> IntersectWithValidity(const arrow::Array& array,
+ const VisibleRanges&
visible_ranges,
+ const arrow::Field& field) {
+ const bool nullable = field.nullable();
+ if (visible_ranges.empty()) {
+ return VisibleRanges{};
+ }
+
+ const int64_t null_count = array.null_count();
+ if (null_count == 0) {
+ return visible_ranges;
+ }
+ if (null_count == array.length()) {
+ if (!nullable) {
+ return NullabilityMismatch(field);
+ }
+ return VisibleRanges{};
+ }
+
+ VisibleRanges valid_ranges;
+ for (const VisibleRange& range : visible_ranges) {
+ int64_t run_start = -1;
+ const int64_t range_end = range.offset + range.length;
+ for (int64_t i = range.offset; i < range_end; ++i) {
+ if (array.IsValid(i)) {
+ if (run_start == -1) {
+ run_start = i;
+ }
+ } else {
+ if (!nullable) {
+ return NullabilityMismatch(field);
+ }
+ if (run_start != -1) {
+ AppendVisibleRange(run_start, i - run_start,
&valid_ranges);
+ run_start = -1;
+ }
+ }
+ }
+ if (run_start != -1) {
+ AppendVisibleRange(run_start, range_end - run_start,
&valid_ranges);
+ }
+ }
+ return valid_ranges;
+}
+
+template <typename ListArray>
+VisibleRanges GetVisibleValueRanges(const ListArray& array,
+ const VisibleRanges&
visible_parent_ranges) {
+ VisibleRanges value_ranges;
+ for (const VisibleRange& range : visible_parent_ranges) {
+ int64_t value_offset = array.value_offset(range.offset);
+ int64_t value_end = array.value_offset(range.offset + range.length);
+ AppendVisibleRange(value_offset, value_end - value_offset,
&value_ranges);
+ }
+ return value_ranges;
+}
+
+Status CheckFieldNullability(const std::shared_ptr<arrow::Field>& field,
+ const std::shared_ptr<arrow::Array>& data,
+ const VisibleRanges& visible_ranges) {
+ const std::shared_ptr<arrow::DataType>& type = field->type();
+ const bool needs_child_validation =
+ type->id() == arrow::Type::STRUCT || type->id() == arrow::Type::LIST ||
+ type->id() == arrow::Type::FIXED_SIZE_LIST || type->id() ==
arrow::Type::MAP;
+ if (!needs_child_validation &&
+ (field->nullable() || visible_ranges.empty() || data->null_count() ==
0)) {
+ return Status::OK();
+ }
+
+ PAIMON_ASSIGN_OR_RAISE(VisibleRanges valid_ranges,
+ IntersectWithValidity(*data, visible_ranges,
*field));
+
+ if (type->id() == arrow::Type::STRUCT) {
+ auto struct_type = checked_pointer_cast<arrow::StructType>(type);
+ auto struct_array = checked_pointer_cast<arrow::StructArray>(data);
+ for (int32_t i = 0; i < struct_type->num_fields(); ++i) {
+ PAIMON_RETURN_NOT_OK(
+ CheckFieldNullability(struct_type->field(i),
struct_array->field(i), valid_ranges));
+ }
+ } else if (type->id() == arrow::Type::LIST) {
+ auto list_type = checked_pointer_cast<arrow::ListType>(type);
+ auto list_array = checked_pointer_cast<arrow::ListArray>(data);
+ VisibleRanges value_ranges = GetVisibleValueRanges(*list_array,
valid_ranges);
+ PAIMON_RETURN_NOT_OK(
+ CheckFieldNullability(list_type->value_field(),
list_array->values(), value_ranges));
+ } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
+ auto vector_type =
checked_pointer_cast<arrow::FixedSizeListType>(type);
+ auto vector_array =
checked_pointer_cast<arrow::FixedSizeListArray>(data);
+ int32_t vector_length = vector_type->list_size();
+ int64_t required_values = (vector_array->offset() +
vector_array->length()) * vector_length;
+ if (vector_array->values()->length() < required_values) {
+ return Status::Invalid(fmt::format(
+ "VECTOR field {} is invalid: VECTOR holds {} elements while {}
rows of dimension "
+ "{} require {}",
+ field->name(), vector_array->values()->length(),
vector_array->length(),
+ vector_length, required_values));
+ }
+
+ VisibleRanges value_ranges = GetVisibleValueRanges(*vector_array,
valid_ranges);
+ // Paimon VECTOR values cannot contain null elements, irrespective of
the Arrow child
+ // field's declared nullability.
+ std::shared_ptr<arrow::Field> value_field =
vector_type->value_field()->WithNullable(false);
+ Status status = CheckFieldNullability(value_field,
vector_array->values(), value_ranges);
+ if (!status.ok()) {
+ return Status::Invalid(
+ fmt::format("VECTOR field {} is invalid: {}", field->name(),
status.message()));
+ }
+ } else if (type->id() == arrow::Type::MAP) {
+ auto map_type = checked_pointer_cast<arrow::MapType>(type);
+ auto map_array = checked_pointer_cast<arrow::MapArray>(data);
+ VisibleRanges value_ranges = GetVisibleValueRanges(*map_array,
valid_ranges);
+ PAIMON_RETURN_NOT_OK(
+ CheckFieldNullability(map_type->key_field(), map_array->keys(),
value_ranges));
+ PAIMON_RETURN_NOT_OK(
+ CheckFieldNullability(map_type->item_field(), map_array->items(),
value_ranges));
+ }
+ return Status::OK();
+}
+
bool NeedsNormalization(const std::shared_ptr<arrow::ArrayData>& data) {
if (data->offset != 0) {
return true;
@@ -407,7 +553,9 @@ Status ArrowUtils::CheckNullabilityMatch(const
std::shared_ptr<arrow::Schema>& s
struct_array->num_fields(), schema->num_fields()));
}
for (int32_t i = 0; i < schema->num_fields(); i++) {
- PAIMON_RETURN_NOT_OK(InnerCheckNullabilityMatch(schema->field(i),
struct_array->field(i)));
+ const std::shared_ptr<arrow::Array>& field_array =
struct_array->field(i);
+ PAIMON_RETURN_NOT_OK(
+ CheckFieldNullability(schema->field(i), field_array, {{0,
field_array->length()}}));
}
return Status::OK();
}
@@ -533,42 +681,6 @@ bool ArrowUtils::EqualsIgnoreNullable(const
std::shared_ptr<arrow::DataType>& ty
return true;
}
-Status ArrowUtils::InnerCheckNullabilityMatch(const
std::shared_ptr<arrow::Field>& field,
- const
std::shared_ptr<arrow::Array>& data) {
- if (PAIMON_UNLIKELY(!field->nullable() && data->null_count() != 0)) {
- return Status::Invalid(fmt::format(
- "CheckNullabilityMatch failed, field {} not nullable while data
have null value",
- field->name()));
- }
- auto type = field->type();
- if (type->id() == arrow::Type::STRUCT) {
- auto struct_type =
checked_pointer_cast<arrow::StructType>(field->type());
- auto struct_array = checked_pointer_cast<arrow::StructArray>(data);
- for (int32_t i = 0; i < struct_type->num_fields(); ++i) {
- PAIMON_RETURN_NOT_OK(
- InnerCheckNullabilityMatch(struct_type->field(i),
struct_array->field(i)));
- }
- } else if (type->id() == arrow::Type::LIST) {
- auto list_type = checked_pointer_cast<arrow::ListType>(field->type());
- auto list_array = checked_pointer_cast<arrow::ListArray>(data);
- PAIMON_RETURN_NOT_OK(
- InnerCheckNullabilityMatch(list_type->value_field(),
list_array->values()));
- } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
- Status status = VectorUtils::ValidateVectorElements(*data);
- if (!status.ok()) {
- return Status::Invalid(
- fmt::format("VECTOR field {} is invalid: {}", field->name(),
status.message()));
- }
- } else if (type->id() == arrow::Type::MAP) {
- auto map_type = checked_pointer_cast<arrow::MapType>(field->type());
- auto map_array = checked_pointer_cast<arrow::MapArray>(data);
- PAIMON_RETURN_NOT_OK(InnerCheckNullabilityMatch(map_type->key_field(),
map_array->keys()));
- PAIMON_RETURN_NOT_OK(
- InnerCheckNullabilityMatch(map_type->item_field(),
map_array->items()));
- }
- return Status::OK();
-}
-
Result<std::shared_ptr<arrow::StructArray>>
ArrowUtils::RemoveFieldFromStructArray(
const std::shared_ptr<arrow::StructArray>& struct_array, const
std::string& field_name) {
auto struct_type =
checked_pointer_cast<arrow::StructType>(struct_array->type());
diff --git a/src/paimon/common/utils/arrow/arrow_utils.h
b/src/paimon/common/utils/arrow/arrow_utils.h
index 2a213155..ac1bfd31 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.h
+++ b/src/paimon/common/utils/arrow/arrow_utils.h
@@ -43,6 +43,8 @@ class PAIMON_EXPORT ArrowUtils {
static Result<std::vector<int32_t>> CreateProjection(
const std::shared_ptr<arrow::Schema>& src_schema, const
arrow::FieldVector& target_fields);
+ /// Checks that every logically visible null matches the schema. Values
hidden by a null
+ /// STRUCT, LIST, MAP, or FIXED_SIZE_LIST parent are ignored.
static Status CheckNullabilityMatch(const std::shared_ptr<arrow::Schema>&
schema,
const std::shared_ptr<arrow::Array>&
data);
@@ -149,10 +151,6 @@ class PAIMON_EXPORT ArrowUtils {
const std::shared_ptr<arrow::StructArray>& batch,
const std::shared_ptr<arrow::DataType>& logical_type,
arrow::MemoryPool* pool,
bool preserve_layout_recoverable_dictionaries);
-
- private:
- static Status InnerCheckNullabilityMatch(const
std::shared_ptr<arrow::Field>& field,
- const
std::shared_ptr<arrow::Array>& data);
};
} // namespace paimon
diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp
b/src/paimon/common/utils/arrow/arrow_utils_test.cpp
index ea3fd1e3..6625e15e 100644
--- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp
@@ -222,6 +222,186 @@ TEST(ArrowUtilsTest, TestCheckNullableMatchWithStruct) {
}
}
+TEST(ArrowUtilsTest, TestCheckNullableMatchParentNullMasksChildren) {
+ auto child = arrow::field("child", arrow::int32(), /*nullable=*/false);
+ auto struct_field = arrow::field("parent", arrow::struct_({child}),
/*nullable=*/true);
+ auto vector_type =
+ arrow::fixed_size_list(arrow::field("item", arrow::float32(),
/*nullable=*/false), 3);
+ auto vector_field = arrow::field("embedding", vector_type,
/*nullable=*/true);
+ auto schema = arrow::schema({struct_field, vector_field});
+
+ // Do not build the complete batch with ArrayFromJSON: for a null STRUCT
it appends a valid
+ // default value to primitive children, which cannot represent the hidden
child null under test.
+ auto parent_with_validity = checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({child}),
R"([null, [1]])")
+ .ValueOrDie());
+ std::shared_ptr<arrow::Array> child_array =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[null,
1]").ValueOrDie();
+ std::shared_ptr<arrow::StructArray> parent =
+ arrow::StructArray::Make({child_array}, {child},
parent_with_validity->null_bitmap(),
+ parent_with_validity->null_count())
+ .ValueOrDie();
+ auto vector = checked_pointer_cast<arrow::FixedSizeListArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(vector_type, R"([null, [1.0,
2.0, 3.0]])")
+ .ValueOrDie());
+ std::shared_ptr<arrow::StructArray> batch =
+ arrow::StructArray::Make({parent, vector}, {struct_field,
vector_field}).ValueOrDie();
+
+ ASSERT_OK(ArrowUtils::CheckNullabilityMatch(schema, batch));
+}
+
+TEST(ArrowUtilsTest, TestCheckNullableMatchRejectsVisibleChildNulls) {
+ auto child = arrow::field("child", arrow::int32(), /*nullable=*/false);
+ auto struct_field = arrow::field("parent", arrow::struct_({child}),
/*nullable=*/true);
+ std::shared_ptr<arrow::Array> struct_batch =
+
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({struct_field}),
R"([[[null]]])")
+ .ValueOrDie();
+ ASSERT_NOK_WITH_MSG(
+ ArrowUtils::CheckNullabilityMatch(arrow::schema({struct_field}),
struct_batch),
+ "CheckNullabilityMatch failed, field child not nullable while data
have null value");
+
+ auto vector_type =
+ arrow::fixed_size_list(arrow::field("item", arrow::float32(),
/*nullable=*/false), 3);
+ auto vector_field = arrow::field("embedding", vector_type,
/*nullable=*/true);
+ std::shared_ptr<arrow::Array> vector_batch =
+
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({vector_field}),
+ R"([[[1.0, null, 3.0]]])")
+ .ValueOrDie();
+ ASSERT_NOK_WITH_MSG(
+ ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}),
vector_batch),
+ "VECTOR field embedding is invalid: CheckNullabilityMatch failed,
field item not nullable "
+ "while data have null value");
+}
+
+TEST(ArrowUtilsTest, TestCheckNullableMatchListParentNullMasksChildren) {
+ auto value_field = arrow::field("value", arrow::int32(),
/*nullable=*/false);
+ auto list_type = arrow::list(value_field);
+ auto list_field = arrow::field("list_column", list_type,
/*nullable=*/true);
+ auto offsets =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1,
2]").ValueOrDie();
+ auto validity_source = checked_pointer_cast<arrow::ListArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(list_type, "[null,
[1]]").ValueOrDie());
+
+ std::shared_ptr<arrow::Array> hidden_values =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[null,
1]").ValueOrDie();
+ std::shared_ptr<arrow::ListArray> hidden_list =
+ arrow::ListArray::FromArrays(list_type, *offsets, *hidden_values,
+ arrow::default_memory_pool(),
validity_source->null_bitmap(),
+ /*null_count=*/1)
+ .ValueOrDie();
+ std::shared_ptr<arrow::StructArray> hidden_batch =
+ arrow::StructArray::Make({hidden_list}, {list_field}).ValueOrDie();
+ ASSERT_TRUE(hidden_batch->ValidateFull().ok());
+ ASSERT_OK(ArrowUtils::CheckNullabilityMatch(arrow::schema({list_field}),
hidden_batch));
+
+ std::shared_ptr<arrow::Array> visible_values =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[null,
null]").ValueOrDie();
+ std::shared_ptr<arrow::ListArray> visible_list =
+ arrow::ListArray::FromArrays(list_type, *offsets, *visible_values,
+ arrow::default_memory_pool(),
validity_source->null_bitmap(),
+ /*null_count=*/1)
+ .ValueOrDie();
+ std::shared_ptr<arrow::StructArray> visible_batch =
+ arrow::StructArray::Make({visible_list}, {list_field}).ValueOrDie();
+ ASSERT_TRUE(visible_batch->ValidateFull().ok());
+ ASSERT_NOK_WITH_MSG(
+ ArrowUtils::CheckNullabilityMatch(arrow::schema({list_field}),
visible_batch),
+ "CheckNullabilityMatch failed, field value not nullable while data
have null value");
+}
+
+TEST(ArrowUtilsTest, TestCheckNullableMatchMapParentNullMasksChildren) {
+ auto key_field = arrow::field("key", arrow::int32(), /*nullable=*/false);
+ auto item_field = arrow::field("item", arrow::int32(), /*nullable=*/false);
+ auto map_type = std::make_shared<arrow::MapType>(key_field, item_field);
+ auto map_field = arrow::field("map_column", map_type, /*nullable=*/true);
+ auto offsets =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1,
2]").ValueOrDie();
+ std::shared_ptr<arrow::Array> keys =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[1,
2]").ValueOrDie();
+ auto validity_source = checked_pointer_cast<arrow::ListArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::list(arrow::int32()),
"[null, [1]]")
+ .ValueOrDie());
+
+ // Arrow rejects physical null MAP keys during construction, so exercise
masking with the
+ // non-nullable item field.
+ std::shared_ptr<arrow::Array> hidden_items =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[null,
20]").ValueOrDie();
+ std::shared_ptr<arrow::Array> hidden_map =
+ arrow::MapArray::FromArrays(map_type, offsets, keys, hidden_items,
+ arrow::default_memory_pool(),
validity_source->null_bitmap())
+ .ValueOrDie();
+ std::shared_ptr<arrow::StructArray> hidden_batch =
+ arrow::StructArray::Make({hidden_map}, {map_field}).ValueOrDie();
+ ASSERT_TRUE(hidden_batch->ValidateFull().ok());
+ ASSERT_OK(ArrowUtils::CheckNullabilityMatch(arrow::schema({map_field}),
hidden_batch));
+
+ std::shared_ptr<arrow::Array> visible_items =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[null,
null]").ValueOrDie();
+ std::shared_ptr<arrow::Array> visible_map =
+ arrow::MapArray::FromArrays(map_type, offsets, keys, visible_items,
+ arrow::default_memory_pool(),
validity_source->null_bitmap())
+ .ValueOrDie();
+ std::shared_ptr<arrow::StructArray> visible_batch =
+ arrow::StructArray::Make({visible_map}, {map_field}).ValueOrDie();
+ ASSERT_TRUE(visible_batch->ValidateFull().ok());
+ ASSERT_NOK_WITH_MSG(
+ ArrowUtils::CheckNullabilityMatch(arrow::schema({map_field}),
visible_batch),
+ "CheckNullabilityMatch failed, field item not nullable while data have
null value");
+}
+
+TEST(ArrowUtilsTest, TestCheckNullableMatchWithSlicedList) {
+ auto value_field = arrow::field("value", arrow::int32(),
/*nullable=*/false);
+ auto list_type = arrow::list(value_field);
+ auto list_field = arrow::field("list_column", list_type,
/*nullable=*/true);
+ auto offsets =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1, 2,
3]").ValueOrDie();
+ std::shared_ptr<arrow::Array> values =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[10, null,
30]").ValueOrDie();
+ auto validity_source = checked_pointer_cast<arrow::ListArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(list_type, "[[10], null,
[30]]").ValueOrDie());
+ std::shared_ptr<arrow::ListArray> list =
+ arrow::ListArray::FromArrays(list_type, *offsets, *values,
arrow::default_memory_pool(),
+ validity_source->null_bitmap(),
/*null_count=*/1)
+ .ValueOrDie();
+ std::shared_ptr<arrow::Array> sliced_list = list->Slice(/*offset=*/1,
/*length=*/2);
+ ASSERT_EQ(sliced_list->offset(), 1);
+
+ std::shared_ptr<arrow::StructArray> batch =
+ arrow::StructArray::Make({sliced_list}, {list_field}).ValueOrDie();
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ ASSERT_OK(ArrowUtils::CheckNullabilityMatch(arrow::schema({list_field}),
batch));
+}
+
+TEST(ArrowUtilsTest, TestCheckNullableMatchDeepParentNullMasksChildren) {
+ auto leaf_field = arrow::field("leaf", arrow::int32(), /*nullable=*/false);
+ auto item_type = arrow::struct_({leaf_field});
+ auto item_field = arrow::field("item", item_type, /*nullable=*/false);
+ auto list_type = arrow::list(item_field);
+ auto list_field = arrow::field("list_column", list_type,
/*nullable=*/true);
+ auto outer_field = arrow::field("outer", arrow::struct_({list_field}),
/*nullable=*/false);
+
+ std::shared_ptr<arrow::Array> leaves =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[null,
1]").ValueOrDie();
+ std::shared_ptr<arrow::StructArray> items =
+ arrow::StructArray::Make({leaves}, {leaf_field}).ValueOrDie();
+ auto offsets =
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1,
2]").ValueOrDie();
+ auto validity_source = checked_pointer_cast<arrow::ListArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::list(arrow::int32()),
"[null, [1]]")
+ .ValueOrDie());
+ std::shared_ptr<arrow::ListArray> list =
+ arrow::ListArray::FromArrays(list_type, *offsets, *items,
arrow::default_memory_pool(),
+ validity_source->null_bitmap(),
/*null_count=*/1)
+ .ValueOrDie();
+ std::shared_ptr<arrow::StructArray> outer =
+ arrow::StructArray::Make({list}, {list_field}).ValueOrDie();
+ std::shared_ptr<arrow::StructArray> batch =
+ arrow::StructArray::Make({outer}, {outer_field}).ValueOrDie();
+
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ ASSERT_OK(ArrowUtils::CheckNullabilityMatch(arrow::schema({outer_field}),
batch));
+}
+
TEST(ArrowUtilsTest, TestCheckNullableMatchWithList) {
auto value_field = arrow::field("value", arrow::int32(),
/*nullable=*/false);
auto list_field = arrow::field("list_column", arrow::list(value_field),
/*nullable=*/false);
@@ -272,7 +452,8 @@ TEST(ArrowUtilsTest,
TestCheckNullableMatchRejectsNullVectorElement) {
ASSERT_NOK_WITH_MSG(
ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}),
struct_array),
- "VECTOR field embedding is invalid: VECTOR cannot contain null
elements");
+ "VECTOR field embedding is invalid: CheckNullabilityMatch failed,
field item not nullable "
+ "while data have null value");
}
// Arrow accepts a FixedSizeList whose child is shorter than `length *
list_size` when importing
