HuaHuaY commented on code in PR #50271:
URL: https://github.com/apache/arrow/pull/50271#discussion_r3550519652
##########
cpp/src/parquet/arrow/reader.cc:
##########
@@ -725,13 +725,66 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public
ListReader<int32_t> {
DCHECK_EQ(data->buffers.size(), 2);
DCHECK_EQ(field()->type()->id(), ::arrow::Type::FIXED_SIZE_LIST);
const auto& type =
checked_cast<::arrow::FixedSizeListType&>(*field()->type());
- const int32_t* offsets = reinterpret_cast<const
int32_t*>(data->buffers[1]->data());
- for (int x = 1; x <= data->length; x++) {
- int32_t size = offsets[x] - offsets[x - 1];
- if (size != type.list_size()) {
- return Status::Invalid("Expected all lists to be of size=",
type.list_size(),
- " but index ", x, " had size=", size);
+ const auto* offsets = reinterpret_cast<const
int32_t*>(data->buffers[1]->data());
+ const int32_t list_size = type.list_size();
+ const uint8_t* valid_bits =
+ data->buffers[0] != nullptr ? data->buffers[0]->data() : nullptr;
+ auto validate_offsets = [&](int64_t start, int64_t length,
+ int32_t expected_size) -> Status {
+ std::span<const int32_t> run_offsets(offsets + start,
+ static_cast<size_t>(length + 1));
+ const auto first_invalid_offset = std::ranges::adjacent_find(
Review Comment:
The newly CI tests have also been compiled, so I haven't changed the code
yet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
