1fanwang commented on code in PR #55704:
URL: https://github.com/apache/arrow/pull/55704#discussion_r4652088416
##########
cpp/src/parquet/arrow/reader.cc:
##########
@@ -785,18 +787,41 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public
ListReader<int32_t> {
::arrow::MakeArray(data->child_data[0]->Slice(offsets[start],
child_length)));
return Status::OK();
};
-
- DCHECK_NE(data->buffers[0], nullptr);
- RETURN_NOT_OK(::arrow::internal::VisitBitRuns(
- data->buffers[0]->data(), data->offset, data->length, visit_run));
+ RETURN_NOT_OK(visit_runs(visit_run));
// TODO(GH-50271): Build one padded child array directly instead of
creating
// one temporary Array/ArrayData per validity run and concatenating them.
ARROW_ASSIGN_OR_RAISE(auto child_array_with_padding,
::arrow::Concatenate(child_arrays, ctx_->pool));
data->child_data[0] = child_array_with_padding->data();
- } else {
- RETURN_NOT_OK(validate_offsets(/*start=*/0, data->length,
/*valid=*/true));
+ return Status::OK();
+ };
+
+ if (data->GetNullCount() != 0) {
+ DCHECK_NE(data->buffers[0], nullptr);
+ RETURN_NOT_OK(rebuild_from_runs([&](auto&& visit_run) {
+ return ::arrow::internal::VisitBitRuns(data->buffers[0]->data(),
data->offset,
+ data->length, visit_run);
+ }));
+ } else if (data->length > 0) {
Review Comment:
Done in a4d4fa94cc. The non-nullable path now validates every run first
without allocating anything, and only rebuilds and concatenates when a run is
actually missing its elements.
##########
cpp/src/parquet/arrow/reader.cc:
##########
@@ -785,18 +787,41 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public
ListReader<int32_t> {
::arrow::MakeArray(data->child_data[0]->Slice(offsets[start],
child_length)));
return Status::OK();
};
-
- DCHECK_NE(data->buffers[0], nullptr);
- RETURN_NOT_OK(::arrow::internal::VisitBitRuns(
- data->buffers[0]->data(), data->offset, data->length, visit_run));
+ RETURN_NOT_OK(visit_runs(visit_run));
// TODO(GH-50271): Build one padded child array directly instead of
creating
// one temporary Array/ArrayData per validity run and concatenating them.
ARROW_ASSIGN_OR_RAISE(auto child_array_with_padding,
::arrow::Concatenate(child_arrays, ctx_->pool));
data->child_data[0] = child_array_with_padding->data();
- } else {
- RETURN_NOT_OK(validate_offsets(/*start=*/0, data->length,
/*valid=*/true));
+ return Status::OK();
+ };
+
+ if (data->GetNullCount() != 0) {
+ DCHECK_NE(data->buffers[0], nullptr);
+ RETURN_NOT_OK(rebuild_from_runs([&](auto&& visit_run) {
+ return ::arrow::internal::VisitBitRuns(data->buffers[0]->data(),
data->offset,
+ data->length, visit_run);
+ }));
+ } else if (data->length > 0) {
+ // This field is schema non-nullable, so it has no validity buffer of
+ // its own: a null ancestor's "elements never written" gap can only be
+ // detected from the offsets directly, not from a (nonexistent) bitmap.
+ RETURN_NOT_OK(rebuild_from_runs([&](auto&& visit_run) -> Status {
+ int64_t run_start = 0;
+ bool run_has_elements = true;
+ for (int64_t i = 0; i < data->length; ++i) {
+ const bool has_elements = offsets[i + 1] != offsets[i];
Review Comment:
Done in a4d4fa94cc. The branch now checks field()->nullable() instead of
buffer existence, so a nullable field with zero nulls validates the whole range
in one pass instead of reaching the padding branch. Added
FixedSizeListRejectsPresentEmptyListWithZeroNulls, built with the low-level
writer API since a real Arrow write can't produce this file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]