wjones127 commented on code in PR #34107:
URL: https://github.com/apache/arrow/pull/34107#discussion_r1110335776
##########
cpp/src/parquet/column_writer.cc:
##########
@@ -1484,6 +1484,39 @@ Status
TypedColumnWriterImpl<DType>::WriteArrowDictionary(
std::shared_ptr<::arrow::Array> dictionary = data.dictionary();
std::shared_ptr<::arrow::Array> indices = data.indices();
+ auto update_stats = [&](int64_t num_chunk_levels,
+ const std::shared_ptr<Array>& chunk_indices) {
+ // TODO(PARQUET-2068) This approach may make two copies. First, a copy of
the
+ // indices array to a (hopefully smaller) referenced indices array.
Second, a copy
+ // of the values array to a (probably not smaller) referenced values array.
+ //
+ // Once the MinMax kernel supports all data types we should use that
kernel instead
+ // as it does not make any copies.
+ ::arrow::compute::ExecContext exec_ctx(ctx->memory_pool);
+ exec_ctx.set_use_threads(false);
+
+ std::shared_ptr<::arrow::Array> referenced_dictionary;
+ PARQUET_ASSIGN_OR_THROW(::arrow::Datum referenced_indices,
+ ::arrow::compute::Unique(*chunk_indices,
&exec_ctx));
+
+ // On first run, we might be able to re-use the existing dictionary
+ if (referenced_indices.length() == dictionary->length()) {
+ referenced_dictionary = dictionary;
+ } else {
+ PARQUET_ASSIGN_OR_THROW(
+ ::arrow::Datum referenced_dictionary_datum,
+ ::arrow::compute::Take(dictionary, referenced_indices,
+
::arrow::compute::TakeOptions(/*boundscheck=*/false),
+ &exec_ctx));
+ referenced_dictionary = referenced_dictionary_datum.make_array();
+ }
+
+ int64_t non_null_count = chunk_indices->length() -
chunk_indices->null_count();
+ page_statistics_->IncrementNullCount(num_chunk_levels - non_null_count);
Review Comment:
Question: is the reason this isn't just set to `chunk_indices->null_count()`
that the null count stats isn't meant to include cases where higher levels
(parent fields) are null? Or something else?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
