mapleFU commented on code in PR #44252:
URL: https://github.com/apache/arrow/pull/44252#discussion_r1827860671
##########
cpp/src/arrow/record_batch.cc:
##########
@@ -465,6 +470,220 @@ Result<std::shared_ptr<RecordBatch>>
RecordBatch::ViewOrCopyTo(
return Make(schema_, num_rows(), std::move(copied_columns));
}
+namespace {
+struct EnumeratedStatistics {
+ int nth_statistics = 0;
+ bool start_new_column = false;
+ std::optional<int32_t> nth_column = std::nullopt;
+ const char* key = nullptr;
+ std::shared_ptr<DataType> type = nullptr;
+ ArrayStatistics::ValueType value = false;
+};
+using OnStatistics =
+ std::function<Status(const EnumeratedStatistics& enumerated_statistics)>;
+Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics
on_statistics) {
+ EnumeratedStatistics statistics;
+ statistics.nth_statistics = 0;
+ statistics.start_new_column = true;
+ statistics.nth_column = std::nullopt;
+
+ statistics.key = ARROW_STATISTICS_KEY_ROW_COUNT_EXACT;
+ statistics.type = int64();
+ statistics.value = record_batch.num_rows();
+ RETURN_NOT_OK(on_statistics(statistics));
+ statistics.start_new_column = false;
+
+ const auto num_fields = record_batch.schema()->num_fields();
+ for (int nth_column = 0; nth_column < num_fields; ++nth_column) {
+ auto column_statistics = record_batch.column(nth_column)->statistics();
+ if (!column_statistics) {
+ continue;
+ }
+
+ statistics.start_new_column = true;
+ statistics.nth_column = nth_column;
+ if (column_statistics->null_count.has_value()) {
+ statistics.nth_statistics++;
+ statistics.key = ARROW_STATISTICS_KEY_NULL_COUNT_EXACT;
+ statistics.type = int64();
+ statistics.value = column_statistics->null_count.value();
+ RETURN_NOT_OK(on_statistics(statistics));
+ statistics.start_new_column = false;
+ }
+
+ if (column_statistics->distinct_count.has_value()) {
+ statistics.nth_statistics++;
+ statistics.key = ARROW_STATISTICS_KEY_DISTINCT_COUNT_EXACT;
+ statistics.type = int64();
+ statistics.value = column_statistics->distinct_count.value();
+ RETURN_NOT_OK(on_statistics(statistics));
+ statistics.start_new_column = false;
+ }
+
+ if (column_statistics->min.has_value()) {
+ statistics.nth_statistics++;
+ if (column_statistics->is_min_exact) {
+ statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_EXACT;
+ } else {
+ statistics.key = ARROW_STATISTICS_KEY_MIN_VALUE_APPROXIMATE;
+ }
+ statistics.type = column_statistics->MinArrowType();
+ statistics.value = column_statistics->min.value();
+ RETURN_NOT_OK(on_statistics(statistics));
+ statistics.start_new_column = false;
+ }
+
+ if (column_statistics->max.has_value()) {
+ statistics.nth_statistics++;
+ if (column_statistics->is_max_exact) {
+ statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_EXACT;
+ } else {
+ statistics.key = ARROW_STATISTICS_KEY_MAX_VALUE_APPROXIMATE;
+ }
+ statistics.type = column_statistics->MaxArrowType();
+ statistics.value = column_statistics->max.value();
+ RETURN_NOT_OK(on_statistics(statistics));
+ statistics.start_new_column = false;
+ }
+ }
+ return Status::OK();
+}
+} // namespace
+
+Result<std::shared_ptr<Array>> RecordBatch::MakeStatisticsArray(
+ MemoryPool* memory_pool) const {
+ // Statistics schema:
+ // struct<
+ // column: int32,
+ // statistics: map<
+ // key: dictionary<
+ // indices: int32,
+ // dictionary: utf8,
+ // >,
+ // items: dense_union<...all needed types...>,
+ // >
+ // >
+
+ // Statistics schema doesn't define static dense union type for
+ // values. Each statistics schema have a dense union type that has
+ // needled value types. The following block collects these types.
Review Comment:
So actually this is logically a "set" prepared for `items`?
##########
cpp/src/arrow/record_batch.cc:
##########
@@ -465,6 +470,220 @@ Result<std::shared_ptr<RecordBatch>>
RecordBatch::ViewOrCopyTo(
return Make(schema_, num_rows(), std::move(copied_columns));
}
+namespace {
+struct EnumeratedStatistics {
+ int nth_statistics = 0;
+ bool start_new_column = false;
+ std::optional<int32_t> nth_column = std::nullopt;
+ const char* key = nullptr;
+ std::shared_ptr<DataType> type = nullptr;
+ ArrayStatistics::ValueType value = false;
+};
+using OnStatistics =
+ std::function<Status(const EnumeratedStatistics& enumerated_statistics)>;
+Status EnumerateStatistics(const RecordBatch& record_batch, OnStatistics
on_statistics) {
Review Comment:
So actually this is for a two-phase building, one pass for types, and
one-pass for data?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
