Copilot commented on code in PR #55408:
URL: https://github.com/apache/arrow/pull/55408#discussion_r4587902330


##########
cpp/src/parquet/column_reader.cc:
##########
@@ -1981,8 +1982,8 @@ int64_t TypedRecordReader<DT, 
VS>::SkipRecordsInBufferNonRepeated(int64_t num_re
   // we do not have a buffer, so we need to read them and throw them away.
   // First we need to figure out how many present/not-null values there are.
   const auto values_to_read = static_cast<int32_t>(
-      std::count(def_levels() + levels_position_, def_levels() + 
remaining_levels_pos,
-                 this->max_def_level()));
+      CountLevels(def_levels() + levels_position_,
+                  remaining_levels_pos - levels_position_, 
this->max_def_level()));

Review Comment:
   `CountLevels` is called unqualified here, while other changes in this file 
use `internal::CountLevels` (e.g., around line 1570). For consistency and to 
avoid relying on incidental `using` directives / scope, qualify this call (and 
similar ones below) as `internal::CountLevels` (or add a single intentional 
`using parquet::internal::CountLevels;` in a narrow scope).



##########
cpp/src/parquet/column_reader.cc:
##########
@@ -2163,8 +2162,8 @@ int64_t TypedRecordReader<DT, VS>::DelimitRecords(int64_t 
num_records,
     }
   }
   // Scan definition levels to find number of physical values
-  *values_seen = std::count(def_levels + level, def_levels + levels_position_,
-                            this->max_def_level());
+  *values_seen =
+      CountLevels(def_levels + level, levels_position_ - level, 
this->max_def_level());

Review Comment:
   These calls use unqualified `CountLevels`, but other call sites in the same 
file use `internal::CountLevels`. Please make the namespace qualification 
consistent across the file (prefer `internal::CountLevels` unless there is a 
deliberate `using` declaration in this scope).



##########
cpp/src/parquet/column_reader.cc:
##########
@@ -2147,9 +2148,7 @@ int64_t TypedRecordReader<DT, VS>::DelimitRecords(int64_t 
num_records,
     const int64_t stride =
         std::min(levels_written() - levels_position_, num_records - 
records_read);
     const int64_t position_end = levels_position_ + stride;
-    for (int64_t i = levels_position_; i < position_end; ++i) {
-      records_read += rep_levels[i] == 0;
-    }
+    records_read += CountLevels(rep_levels + levels_position_, stride, 0);

Review Comment:
   These calls use unqualified `CountLevels`, but other call sites in the same 
file use `internal::CountLevels`. Please make the namespace qualification 
consistent across the file (prefer `internal::CountLevels` unless there is a 
deliberate `using` declaration in this scope).



##########
cpp/src/parquet/level_comparison.h:
##########
@@ -35,4 +37,55 @@ struct MinMax {
 
 MinMax PARQUET_EXPORT FindMinMax(const int16_t* levels, int64_t num_levels);
 
+// Fallback for num_levels > uint16_t::max(), counting in chunks to avoid 
overflow.
+template <typename Predicate>
+ARROW_NOINLINE int64_t CountLevelsIfChunked(const int16_t* levels, int64_t 
num_levels,
+                                            Predicate&& pred) {
+  constexpr int64_t kMaxChunkSize = std::numeric_limits<uint16_t>::max();
+  int64_t total = 0;
+  while (num_levels > 0) {
+    const int64_t chunk_size = std::min(num_levels, kMaxChunkSize);
+    uint16_t chunk_count = 0;
+    for (int64_t i = 0; i < chunk_size; ++i) {
+      chunk_count += pred(levels[i]) ? 1 : 0;
+    }
+    total += chunk_count;
+    levels += chunk_size;
+    num_levels -= chunk_size;
+  }
+  return total;
+}

Review Comment:
   The PR description mentions that non-template wrappers allow ODR 
deduplication of the out-of-line fallback across instantiations. However, the 
wrappers currently call the *templated* `CountLevelsIf` with lambdas, which 
results in distinct template instantiations per translation unit (because 
lambda closure types are TU-unique). This can increase code size and undermines 
the stated deduplication benefit. Consider making the long-input fallback a 
non-template function (e.g., specialized `CountLevelsEqChunked(levels, 
num_levels, value)` / `CountLevelsGeChunked(...)` implemented in a `.cc`), or 
otherwise avoid TU-unique predicate types on the fallback path while keeping 
the short fast-path inline.



##########
cpp/src/parquet/column_reader.cc:
##########
@@ -2253,8 +2252,9 @@ void TypedRecordReader<DT, 
VS>::ReadDenseForOptionalInBuffer(
 
   // When reading dense we need to figure out number of values to read.
   const int16_t* def_levels = this->def_levels();
-  *values_to_read += std::count(def_levels + start_levels_position,
-                                def_levels + levels_position_, 
this->max_def_level());
+  *values_to_read +=
+      CountLevels(def_levels + start_levels_position,
+                  levels_position_ - start_levels_position, 
this->max_def_level());

Review Comment:
   Same as above: this unqualified `CountLevels` call is inconsistent with 
other call sites using `internal::CountLevels`. Qualifying it improves 
readability and reduces the chance of accidental name lookup changes.



##########
cpp/src/parquet/level_comparison.h:
##########
@@ -35,4 +37,55 @@ struct MinMax {
 
 MinMax PARQUET_EXPORT FindMinMax(const int16_t* levels, int64_t num_levels);
 
+// Fallback for num_levels > uint16_t::max(), counting in chunks to avoid 
overflow.
+template <typename Predicate>
+ARROW_NOINLINE int64_t CountLevelsIfChunked(const int16_t* levels, int64_t 
num_levels,
+                                            Predicate&& pred) {
+  constexpr int64_t kMaxChunkSize = std::numeric_limits<uint16_t>::max();
+  int64_t total = 0;
+  while (num_levels > 0) {
+    const int64_t chunk_size = std::min(num_levels, kMaxChunkSize);
+    uint16_t chunk_count = 0;
+    for (int64_t i = 0; i < chunk_size; ++i) {
+      chunk_count += pred(levels[i]) ? 1 : 0;
+    }
+    total += chunk_count;
+    levels += chunk_size;
+    num_levels -= chunk_size;
+  }
+  return total;
+}
+
+/// \brief Count the levels that satisfy `pred`.
+///
+/// Accumulates matches in a `uint16_t` so the compiler can vectorize the loop
+/// using 16-bit lanes without widening each comparison result to 64 bits.
+/// Inputs longer than `uint16_t::max()` are counted in chunks to avoid 
overflow.
+template <typename Predicate>
+inline int64_t CountLevelsIf(const int16_t* levels, int64_t num_levels,
+                             Predicate&& pred) {
+  constexpr int64_t kMaxChunkSize = std::numeric_limits<uint16_t>::max();
+  if (num_levels > kMaxChunkSize) {
+    return CountLevelsIfChunked(levels, num_levels, 
std::forward<Predicate>(pred));
+  }
+  uint16_t count = 0;
+  for (int64_t i = 0; i < num_levels; ++i) {
+    count += pred(levels[i]) ? 1 : 0;
+  }
+  return count;
+}
+
+/// \brief Count the levels equal to `value`.
+inline int64_t CountLevels(const int16_t* levels, int64_t num_levels, int16_t 
value) {
+  return CountLevelsIf(levels, num_levels,
+                       [value](int16_t level) { return level == value; });
+}
+
+/// \brief Count the levels greater than or equal to `min_level`.
+inline int64_t CountLevelsGreaterEqual(const int16_t* levels, int64_t 
num_levels,
+                                       int16_t min_level) {
+  return CountLevelsIf(levels, num_levels,
+                       [min_level](int16_t level) { return level >= min_level; 
});
+}

Review Comment:
   The PR description mentions that non-template wrappers allow ODR 
deduplication of the out-of-line fallback across instantiations. However, the 
wrappers currently call the *templated* `CountLevelsIf` with lambdas, which 
results in distinct template instantiations per translation unit (because 
lambda closure types are TU-unique). This can increase code size and undermines 
the stated deduplication benefit. Consider making the long-input fallback a 
non-template function (e.g., specialized `CountLevelsEqChunked(levels, 
num_levels, value)` / `CountLevelsGeChunked(...)` implemented in a `.cc`), or 
otherwise avoid TU-unique predicate types on the fallback path while keeping 
the short fast-path inline.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to