HuaHuaY commented on code in PR #50807:
URL: https://github.com/apache/arrow/pull/50807#discussion_r3805047000
##########
cpp/src/parquet/statistics.cc:
##########
@@ -870,18 +1112,85 @@ class TypedStatisticsImpl : public
TypedStatistics<DType> {
this->has_distinct_count_ = false;
// Null count calculation is cheap and enabled by default.
this->has_null_count_ = true;
+ // NaN counts are collected alongside floating-point bounds and enabled by
+ // default.
+ if constexpr (std::same_as<DType, FloatType> || std::same_as<DType,
DoubleType>) {
+ this->has_nan_count_ = true;
+ } else if constexpr (std::same_as<DType, FLBAType>) {
+ this->has_nan_count_ = is_half_float_;
+ } else {
+ this->has_nan_count_ = false;
+ }
+ }
+
+ template <ColumnOrder::type column_order, typename VisitValues>
+ void UpdateFloatingBoundsWithOrder(VisitValues&& visit_values, bool
update_nan_count) {
+ using ArrowFloat = decltype(ToArrowFloat(std::declval<T>()));
+
+ FloatingValueSummary<ArrowFloat, column_order> summary;
+ std::invoke(std::forward<VisitValues>(visit_values),
+ [&](const auto& value) { summary.Add(value); });
+ if (has_nan_count_ && update_nan_count) {
+ statistics_.nan_count += summary.nan_count();
+ }
+ const auto& bounds = summary.bounds();
+ if (bounds.has_value()) {
+ if constexpr (std::same_as<DType, FLBAType>) {
+ DCHECK(is_half_float_);
+ const auto min = bounds->first.ToLittleEndian();
+ const auto max = bounds->second.ToLittleEndian();
+ SetMinMaxPair({FLBA{min.data()}, FLBA{max.data()}});
+ } else {
+ SetMinMaxPair(bounds.value());
+ }
+ }
+ }
+
+ template <typename VisitValues>
+ void UpdateFloatingBounds(VisitValues&& visit_values, bool update_nan_count)
{
+ if (descr_->column_order().get_order() ==
ColumnOrder::IEEE_754_TOTAL_ORDER) {
+ UpdateFloatingBoundsWithOrder<ColumnOrder::IEEE_754_TOTAL_ORDER>(
+ std::forward<VisitValues>(visit_values), update_nan_count);
+ } else {
+ DCHECK(descr_->can_use_min_max());
+ UpdateFloatingBoundsWithOrder<ColumnOrder::TYPE_DEFINED_ORDER>(
+ std::forward<VisitValues>(visit_values), update_nan_count);
+ }
}
void SetMinMaxPair(std::pair<T, T> min_max) {
if (comparator_ == nullptr) return;
- // CleanStatistic can return a nullopt in case of erroneous values, e.g.
NaN
- auto maybe_min_max = CleanStatistic(min_max, logical_type_);
+ auto maybe_min_max =
+ descr_->column_order().get_order() == ColumnOrder::IEEE_754_TOTAL_ORDER
+ ? std::optional<std::pair<T, T>>(min_max)
+ : CleanStatistic(min_max, logical_type_);
if (!maybe_min_max) return;
auto min = maybe_min_max.value().first;
auto max = maybe_min_max.value().second;
- if (!has_min_max_) {
+ bool replace_all_nan_bounds = false;
+ if constexpr (std::same_as<DType, FloatType> || std::same_as<DType,
DoubleType> ||
+ std::same_as<DType, FLBAType>) {
+ if (descr_->column_order().get_order() ==
ColumnOrder::IEEE_754_TOTAL_ORDER) {
+ DCHECK((!std::same_as<DType, FLBAType>) || is_half_float_);
+
+ const bool min_is_nan = IsNaNValue(ToArrowFloat(min));
+ DCHECK_EQ(min_is_nan, IsNaNValue(ToArrowFloat(max)));
Review Comment:
The `min`/`max` values ​​generated by `UpdateFloatingBoundsWithOrder` will
not violate this constraint, so I think it's best to compile this check as a
noop under release mode.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
