Copilot commented on code in PR #55408:
URL: https://github.com/apache/arrow/pull/55408#discussion_r4587104947
##########
cpp/src/parquet/column_reader.cc:
##########
@@ -1962,8 +1963,8 @@ int64_t TypedRecordReader<DT,
VS>::SkipRecordsInBufferNonRepeated(int64_t num_re
// we do not have a buffer, so we need to read them and throw them away.
// First we need to figure out how many present/not-null values there are.
const auto values_to_read = static_cast<int32_t>(
- std::count(def_levels() + levels_position_, def_levels() +
remaining_levels_pos,
- this->max_def_level()));
+ CountLevels(def_levels() + levels_position_,
+ remaining_levels_pos - levels_position_,
this->max_def_level()));
Review Comment:
The file mixes `internal::CountLevels(...)` (line 1551) with unqualified
`CountLevels(...)` calls (lines 1966, 2132, 2147, 2237). If these call sites
are not in `parquet::internal` scope (or lack a `using` declaration), this will
fail to compile or resolve unexpectedly. Make the qualification consistent
(e.g., use `internal::CountLevels` everywhere in this file, or add a
clearly-scoped `using parquet::internal::CountLevels;` where appropriate).
##########
cpp/src/parquet/column_reader.cc:
##########
@@ -1547,7 +1548,7 @@ class TypedColumnReaderImpl
// cache-efficiency if fused with the level decoding.
ARROW_DCHECK_LE(*num_def_levels, std::numeric_limits<int32_t>::max());
*non_null_values_to_read += static_cast<int32_t>(
- std::count(def_levels, def_levels + *num_def_levels,
this->max_def_level()));
+ internal::CountLevels(def_levels, *num_def_levels,
this->max_def_level()));
Review Comment:
The file mixes `internal::CountLevels(...)` (line 1551) with unqualified
`CountLevels(...)` calls (lines 1966, 2132, 2147, 2237). If these call sites
are not in `parquet::internal` scope (or lack a `using` declaration), this will
fail to compile or resolve unexpectedly. Make the qualification consistent
(e.g., use `internal::CountLevels` everywhere in this file, or add a
clearly-scoped `using parquet::internal::CountLevels;` where appropriate).
##########
cpp/src/parquet/column_reader.cc:
##########
@@ -2128,9 +2129,7 @@ int64_t TypedRecordReader<DT, VS>::DelimitRecords(int64_t
num_records,
const int64_t stride =
std::min(levels_written() - levels_position_, num_records -
records_read);
const int64_t position_end = levels_position_ + stride;
- for (int64_t i = levels_position_; i < position_end; ++i) {
- records_read += rep_levels[i] == 0;
- }
+ records_read += CountLevels(rep_levels + levels_position_, stride, 0);
Review Comment:
The file mixes `internal::CountLevels(...)` (line 1551) with unqualified
`CountLevels(...)` calls (lines 1966, 2132, 2147, 2237). If these call sites
are not in `parquet::internal` scope (or lack a `using` declaration), this will
fail to compile or resolve unexpectedly. Make the qualification consistent
(e.g., use `internal::CountLevels` everywhere in this file, or add a
clearly-scoped `using parquet::internal::CountLevels;` where appropriate).
##########
cpp/src/parquet/column_reader.cc:
##########
@@ -2144,8 +2143,8 @@ int64_t TypedRecordReader<DT, VS>::DelimitRecords(int64_t
num_records,
}
}
// Scan definition levels to find number of physical values
- *values_seen = std::count(def_levels + level, def_levels + levels_position_,
- this->max_def_level());
+ *values_seen =
+ CountLevels(def_levels + level, levels_position_ - level,
this->max_def_level());
Review Comment:
The file mixes `internal::CountLevels(...)` (line 1551) with unqualified
`CountLevels(...)` calls (lines 1966, 2132, 2147, 2237). If these call sites
are not in `parquet::internal` scope (or lack a `using` declaration), this will
fail to compile or resolve unexpectedly. Make the qualification consistent
(e.g., use `internal::CountLevels` everywhere in this file, or add a
clearly-scoped `using parquet::internal::CountLevels;` where appropriate).
##########
cpp/src/parquet/level_comparison.h:
##########
@@ -35,4 +37,55 @@ struct MinMax {
MinMax PARQUET_EXPORT FindMinMax(const int16_t* levels, int64_t num_levels);
+// Fallback for num_levels > uint16_t::max(), counting in chunks to avoid
overflow.
+template <typename Predicate>
+ARROW_NOINLINE int64_t CountLevelsIfChunked(const int16_t* levels, int64_t
num_levels,
+ Predicate&& pred) {
+ constexpr int64_t kMaxChunkSize = std::numeric_limits<uint16_t>::max();
+ int64_t total = 0;
+ while (num_levels > 0) {
+ const int64_t chunk_size = std::min(num_levels, kMaxChunkSize);
+ uint16_t chunk_count = 0;
+ for (int64_t i = 0; i < chunk_size; ++i) {
+ chunk_count += pred(levels[i]) ? 1 : 0;
+ }
+ total += chunk_count;
+ levels += chunk_size;
+ num_levels -= chunk_size;
+ }
+ return total;
+}
+
+/// \brief Count the levels that satisfy `pred`.
+///
+/// Accumulates matches in a `uint16_t` so the compiler can vectorize the loop
+/// using 16-bit lanes without widening each comparison result to 64 bits.
+/// Inputs longer than `uint16_t::max()` are counted in chunks to avoid
overflow.
+template <typename Predicate>
+inline int64_t CountLevelsIf(const int16_t* levels, int64_t num_levels,
+ Predicate&& pred) {
+ constexpr int64_t kMaxChunkSize = std::numeric_limits<uint16_t>::max();
+ if (num_levels > kMaxChunkSize) {
+ return CountLevelsIfChunked(levels, num_levels,
std::forward<Predicate>(pred));
+ }
Review Comment:
`CountLevelsIfChunked` is a templated function defined in a header; even
with `ARROW_NOINLINE`, it will typically be instantiated separately per
`Predicate` type (and potentially per translation unit), which can increase
compile time and binary size. If the intent is a single truly out-of-line cold
path, consider refactoring the chunked fallback into a non-templated
implementation (e.g., specialized helpers for equality / >=, or an enum-driven
comparator) placed in a `.cc` file, while keeping the small-vector fast path
inline.
##########
cpp/src/parquet/column_reader.cc:
##########
@@ -2234,8 +2233,9 @@ void TypedRecordReader<DT,
VS>::ReadDenseForOptionalInBuffer(
// When reading dense we need to figure out number of values to read.
const int16_t* def_levels = this->def_levels();
- *values_to_read += std::count(def_levels + start_levels_position,
- def_levels + levels_position_,
this->max_def_level());
+ *values_to_read +=
+ CountLevels(def_levels + start_levels_position,
+ levels_position_ - start_levels_position,
this->max_def_level());
Review Comment:
The file mixes `internal::CountLevels(...)` (line 1551) with unqualified
`CountLevels(...)` calls (lines 1966, 2132, 2147, 2237). If these call sites
are not in `parquet::internal` scope (or lack a `using` declaration), this will
fail to compile or resolve unexpectedly. Make the qualification consistent
(e.g., use `internal::CountLevels` everywhere in this file, or add a
clearly-scoped `using parquet::internal::CountLevels;` where appropriate).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]