Copilot commented on code in PR #55704:
URL: https://github.com/apache/arrow/pull/55704#discussion_r4651902910
##########
cpp/src/parquet/arrow/reader.cc:
##########
@@ -785,18 +787,41 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public
ListReader<int32_t> {
::arrow::MakeArray(data->child_data[0]->Slice(offsets[start],
child_length)));
return Status::OK();
};
-
- DCHECK_NE(data->buffers[0], nullptr);
- RETURN_NOT_OK(::arrow::internal::VisitBitRuns(
- data->buffers[0]->data(), data->offset, data->length, visit_run));
+ RETURN_NOT_OK(visit_runs(visit_run));
// TODO(GH-50271): Build one padded child array directly instead of
creating
// one temporary Array/ArrayData per validity run and concatenating them.
ARROW_ASSIGN_OR_RAISE(auto child_array_with_padding,
::arrow::Concatenate(child_arrays, ctx_->pool));
data->child_data[0] = child_array_with_padding->data();
- } else {
- RETURN_NOT_OK(validate_offsets(/*start=*/0, data->length,
/*valid=*/true));
+ return Status::OK();
+ };
+
+ if (data->GetNullCount() != 0) {
+ DCHECK_NE(data->buffers[0], nullptr);
+ RETURN_NOT_OK(rebuild_from_runs([&](auto&& visit_run) {
+ return ::arrow::internal::VisitBitRuns(data->buffers[0]->data(),
data->offset,
+ data->length, visit_run);
+ }));
+ } else if (data->length > 0) {
+ // This field is schema non-nullable, so it has no validity buffer of
+ // its own: a null ancestor's "elements never written" gap can only be
+ // detected from the offsets directly, not from a (nonexistent) bitmap.
+ RETURN_NOT_OK(rebuild_from_runs([&](auto&& visit_run) -> Status {
+ int64_t run_start = 0;
+ bool run_has_elements = true;
+ for (int64_t i = 0; i < data->length; ++i) {
+ const bool has_elements = offsets[i + 1] != offsets[i];
Review Comment:
Equal offsets also represent a present empty Parquet list, and nullable
fields with zero nulls reach this branch too. With stored Arrow metadata
declaring fixed size 2, a physical `[]` now becomes `[null, null]` instead of
raising the previous size-mismatch error. Use definition-level or
ancestor-validity information to pad only missing slots, and add a regression
that still rejects present empty lists for positive fixed sizes.
##########
cpp/src/parquet/arrow/reader.cc:
##########
@@ -785,18 +787,41 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public
ListReader<int32_t> {
::arrow::MakeArray(data->child_data[0]->Slice(offsets[start],
child_length)));
return Status::OK();
};
-
- DCHECK_NE(data->buffers[0], nullptr);
- RETURN_NOT_OK(::arrow::internal::VisitBitRuns(
- data->buffers[0]->data(), data->offset, data->length, visit_run));
+ RETURN_NOT_OK(visit_runs(visit_run));
// TODO(GH-50271): Build one padded child array directly instead of
creating
// one temporary Array/ArrayData per validity run and concatenating them.
ARROW_ASSIGN_OR_RAISE(auto child_array_with_padding,
::arrow::Concatenate(child_arrays, ctx_->pool));
data->child_data[0] = child_array_with_padding->data();
- } else {
- RETURN_NOT_OK(validate_offsets(/*start=*/0, data->length,
/*valid=*/true));
+ return Status::OK();
+ };
+
+ if (data->GetNullCount() != 0) {
+ DCHECK_NE(data->buffers[0], nullptr);
+ RETURN_NOT_OK(rebuild_from_runs([&](auto&& visit_run) {
+ return ::arrow::internal::VisitBitRuns(data->buffers[0]->data(),
data->offset,
+ data->length, visit_run);
+ }));
+ } else if (data->length > 0) {
Review Comment:
Every nonempty batch with zero list nulls now rebuilds and concatenates its
child, even when every offset span already equals `list_size`.
`arrow::Concatenate` copies primitive value buffers even for a single input
(`array/concatenate.cc:376-383`, `buffer.cc:194-209`), so ordinary
fixed-size-list reads gain a full child-buffer allocation and copy. Please
preserve the validation-only fast path when no padding is needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]