mapleFU commented on code in PR #35758:
URL: https://github.com/apache/arrow/pull/35758#discussion_r1205099210
##########
cpp/src/parquet/metadata.cc:
##########
@@ -1462,40 +1462,45 @@ class
ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl {
column_chunk_->meta_data.__set_total_compressed_size(compressed_size);
std::vector<format::Encoding::type> thrift_encodings;
+ std::vector<format::PageEncodingStats> thrift_encoding_stats;
+ // Force Emplace encoding for RL/DL.
+ thrift_encodings.push_back(ToThrift(Encoding::RLE));
+ // Add dictionary page encoding stats
if (has_dictionary) {
-
thrift_encodings.push_back(ToThrift(properties_->dictionary_index_encoding()));
+ // For Parquet V1, Dictionary Data Page and Dictionary Index Page
encodings
+ // is PLAIN_DICTIONARY, but the actual DATA is PLAIN, so force adding
PLAIN.
if (properties_->version() == ParquetVersion::PARQUET_1_0) {
thrift_encodings.push_back(ToThrift(Encoding::PLAIN));
- } else {
-
thrift_encodings.push_back(ToThrift(properties_->dictionary_page_encoding()));
}
- } else { // Dictionary not enabled
-
thrift_encodings.push_back(ToThrift(properties_->encoding(column_->path())));
- }
- thrift_encodings.push_back(ToThrift(Encoding::RLE));
- // Only PLAIN encoding is supported for fallback in V1
- // TODO(majetideepak): Use user specified encoding for V2
- if (dictionary_fallback) {
- thrift_encodings.push_back(ToThrift(Encoding::PLAIN));
- }
- column_chunk_->meta_data.__set_encodings(thrift_encodings);
- std::vector<format::PageEncodingStats> thrift_encoding_stats;
- // Add dictionary page encoding stats
- for (const auto& entry : dict_encoding_stats) {
- format::PageEncodingStats dict_enc_stat;
- dict_enc_stat.__set_page_type(format::PageType::DICTIONARY_PAGE);
- dict_enc_stat.__set_encoding(ToThrift(entry.first));
- dict_enc_stat.__set_count(entry.second);
- thrift_encoding_stats.push_back(dict_enc_stat);
+ for (const auto& entry : dict_encoding_stats) {
+ format::PageEncodingStats dict_enc_stat;
+ dict_enc_stat.__set_page_type(format::PageType::DICTIONARY_PAGE);
+ format::Encoding::type dict_encoding = ToThrift(entry.first);
Review Comment:
Yes ...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
