wgtmac commented on code in PR #36574:
URL: https://github.com/apache/arrow/pull/36574#discussion_r1302509826
##########
cpp/src/parquet/encryption/test_encryption_util.cc:
##########
@@ -509,4 +513,178 @@ void FileDecryptor::CheckFile(parquet::ParquetFileReader*
file_reader,
}
}
+void FileDecryptor::DecryptPageIndex(
+ const std::string& file,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::string exception_msg;
+ parquet::ReaderProperties reader_properties =
parquet::default_reader_properties();
+ if (file_decryption_properties) {
+
reader_properties.file_decryption_properties(file_decryption_properties->DeepClone());
+ }
+
+ std::shared_ptr<::arrow::io::RandomAccessFile> source;
+ PARQUET_ASSIGN_OR_THROW(
+ source, ::arrow::io::ReadableFile::Open(file,
reader_properties.memory_pool()));
+
+ auto file_reader = parquet::ParquetFileReader::Open(source,
reader_properties);
+ CheckPageIndex(file_reader.get(), file_decryption_properties);
+
+ ASSERT_NO_FATAL_FAILURE(file_reader->Close());
+ PARQUET_THROW_NOT_OK(source->Close());
+}
+
+void FileDecryptor::CheckPageIndex(
+ parquet::ParquetFileReader* file_reader,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::shared_ptr<PageIndexReader> page_index_reader =
file_reader->GetPageIndexReader();
+ ASSERT_NE(page_index_reader, nullptr);
+
+ const std::shared_ptr<parquet::FileMetaData> file_metadata =
file_reader->metadata();
+ const int num_row_groups = file_metadata->num_row_groups();
+ const int num_columns = file_metadata->num_columns();
+ ASSERT_EQ(num_columns, 8);
+
+ // We cannot read page index of encrypted columns in the plaintext mode
+ std::vector<int32_t> need_row_groups(num_row_groups);
+ std::for_each(need_row_groups.begin(), need_row_groups.end(),
+ [i = 0](int32_t& v) mutable { v = i++; });
+ std::vector<int32_t> need_columns;
+ if (file_decryption_properties == nullptr) {
+ need_columns = {0, 1, 2, 3, 6, 7};
+ } else {
+ need_columns = {0, 1, 2, 3, 4, 5, 6, 7};
+ }
+
+ // Provide hint of requested columns to avoid accessing encrypted columns
without
+ // decryption properties.
+ page_index_reader->WillNeed(
+ need_row_groups, need_columns,
+ PageIndexSelection{/*column_index=*/true, /*offset_index=*/true});
+
+ // Iterate over all the RowGroups in the file.
+ for (int r = 0; r < num_row_groups; ++r) {
+ auto row_group_page_index_reader = page_index_reader->RowGroup(r);
+ ASSERT_NE(row_group_page_index_reader, nullptr);
+
+ for (int c = 0; c < num_columns; ++c) {
+ // Skip reading encrypted columns without decryption properties.
Review Comment:
The reason is that PageIndexReader will try to coalesce I/O ranges of
requested columns. This requires decryption of column metadata. Without file
decryption properties, we cannot even read any column here. So I decided to
skip these encrypted columns.
Actually those failing cases are checked in the test here and around:
https://github.com/apache/arrow/pull/36574/commits/6741c7213d99ad892b696629c620e96a2d8398bf#diff-371a1f4f383d941e926614b706b04810a795c4e14bd25ef898371dd38dd54582R213
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
