emkornfield commented on code in PR #40594:
URL: https://github.com/apache/arrow/pull/40594#discussion_r1855238470
##########
cpp/src/parquet/column_writer.cc:
##########
@@ -1554,6 +1591,46 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
public TypedColumnWriter<
rows_written_ += num_levels;
num_buffered_rows_ += num_levels;
}
+
+ UpdateLevelHistogram(num_levels, def_levels, rep_levels);
+ }
+
+ void UpdateLevelHistogram(int64_t num_levels, const int16_t* def_levels,
+ const int16_t* rep_levels) const {
+ if (page_size_statistics_ == nullptr) {
+ return;
+ }
+
+ auto add_levels = [](std::vector<int64_t>& level_histogram,
+ ::arrow::util::span<const int16_t> levels) {
+ for (int16_t level : levels) {
+ ARROW_DCHECK_LT(level, static_cast<int16_t>(level_histogram.size()));
+ ++level_histogram[level];
+ }
+ };
+
+ if (descr_->max_definition_level() > 0) {
+ add_levels(page_size_statistics_->definition_level_histogram,
+ {def_levels, static_cast<size_t>(num_levels)});
+ } else {
+ page_size_statistics_->definition_level_histogram[0] += num_levels;
+ }
+
+ if (descr_->max_repetition_level() > 0) {
+ add_levels(page_size_statistics_->repetition_level_histogram,
+ {rep_levels, static_cast<size_t>(num_levels)});
+ } else {
+ page_size_statistics_->repetition_level_histogram[0] += num_levels;
+ }
+ }
+
+ void UpdateUnencodedDataBytes() const {
+ if constexpr (std::is_same_v<T, ByteArray>) {
Review Comment:
might be worth a comment reminding readers that this is only needed/set for
byte arrays per the specification.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
