pitrou commented on code in PR #45202:
URL: https://github.com/apache/arrow/pull/45202#discussion_r1908447406
##########
cpp/src/parquet/size_statistics.cc:
##########
@@ -91,4 +100,83 @@ std::unique_ptr<SizeStatistics> SizeStatistics::Make(const
ColumnDescriptor* des
return size_stats;
}
+std::ostream& operator<<(std::ostream& os, const SizeStatistics& size_stats) {
+ constexpr std::string_view kComma = ", ";
+ os << "SizeStatistics{";
+ std::string_view sep = "";
+ if (size_stats.unencoded_byte_array_data_bytes.has_value()) {
+ os << "unencoded_byte_array_data_bytes="
+ << *size_stats.unencoded_byte_array_data_bytes;
+ sep = kComma;
+ }
+ auto print_histogram = [&](std::string_view name,
+ const std::vector<int64_t>& histogram) {
+ if (!histogram.empty()) {
+ os << sep << name << "={";
+ sep = kComma;
+ std::string_view value_sep = "";
+ for (int64_t v : histogram) {
+ os << value_sep << v;
+ value_sep = kComma;
+ }
+ os << "}";
+ }
+ };
+ print_histogram("repetition_level_histogram",
size_stats.repetition_level_histogram);
+ print_histogram("definition_level_histogram",
size_stats.definition_level_histogram);
+ os << "}";
+ return os;
+}
+
+void UpdateLevelHistogram(::arrow::util::span<const int16_t> levels,
+ ::arrow::util::span<int64_t> histogram) {
+ const int64_t num_levels = static_cast<int64_t>(levels.size());
+ const int16_t max_level = static_cast<int16_t>(histogram.size() - 1);
+ if (max_level == 0) {
+ histogram[0] += num_levels;
+ return;
+ }
+ // The goal of the two specialized paths below is to accelerate common cases
+ // by keeping histogram values in registers.
+ // The fallback implementation (`++histogram[level]`) issues a series of
+ // load-stores with frequent conflicts.
+ if (max_level == 1) {
+ // Specialize the common case for non-repeated non-nested columns
+ // by keeping histogram values in a register, which avoids being limited
+ // by CPU cache latency.
+ int64_t hist0 = 0;
+ for (int16_t level : levels) {
+ ARROW_DCHECK_LE(level, max_level);
+ hist0 += (level == 0);
Review Comment:
Interesting suggestion, I'll try it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
