pitrou commented on code in PR #36574:
URL: https://github.com/apache/arrow/pull/36574#discussion_r1299833893
##########
cpp/src/parquet/encryption/test_encryption_util.cc:
##########
@@ -509,4 +513,178 @@ void FileDecryptor::CheckFile(parquet::ParquetFileReader*
file_reader,
}
}
+void FileDecryptor::DecryptPageIndex(
+ const std::string& file,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::string exception_msg;
+ parquet::ReaderProperties reader_properties =
parquet::default_reader_properties();
+ if (file_decryption_properties) {
+
reader_properties.file_decryption_properties(file_decryption_properties->DeepClone());
+ }
+
+ std::shared_ptr<::arrow::io::RandomAccessFile> source;
+ PARQUET_ASSIGN_OR_THROW(
+ source, ::arrow::io::ReadableFile::Open(file,
reader_properties.memory_pool()));
+
+ auto file_reader = parquet::ParquetFileReader::Open(source,
reader_properties);
+ CheckPageIndex(file_reader.get(), file_decryption_properties);
+
+ ASSERT_NO_FATAL_FAILURE(file_reader->Close());
+ PARQUET_THROW_NOT_OK(source->Close());
+}
+
+void FileDecryptor::CheckPageIndex(
+ parquet::ParquetFileReader* file_reader,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::shared_ptr<PageIndexReader> page_index_reader =
file_reader->GetPageIndexReader();
+ ASSERT_NE(page_index_reader, nullptr);
+
+ const std::shared_ptr<parquet::FileMetaData> file_metadata =
file_reader->metadata();
+ const int num_row_groups = file_metadata->num_row_groups();
+ const int num_columns = file_metadata->num_columns();
+ ASSERT_EQ(num_columns, 8);
+
+ // We cannot read page index of encrypted columns in the plaintext mode
+ std::vector<int32_t> need_row_groups(num_row_groups);
+ std::for_each(need_row_groups.begin(), need_row_groups.end(),
+ [i = 0](int32_t& v) mutable { v = i++; });
+ std::vector<int32_t> need_columns;
+ if (file_decryption_properties == nullptr) {
+ need_columns = {0, 1, 2, 3, 6, 7};
Review Comment:
I don't understand: only columns 4 and 5 are encrypted? Is this documented
somewhere?
##########
cpp/src/parquet/encryption/test_encryption_util.cc:
##########
@@ -509,4 +513,178 @@ void FileDecryptor::CheckFile(parquet::ParquetFileReader*
file_reader,
}
}
+void FileDecryptor::DecryptPageIndex(
+ const std::string& file,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::string exception_msg;
+ parquet::ReaderProperties reader_properties =
parquet::default_reader_properties();
+ if (file_decryption_properties) {
+
reader_properties.file_decryption_properties(file_decryption_properties->DeepClone());
+ }
+
+ std::shared_ptr<::arrow::io::RandomAccessFile> source;
+ PARQUET_ASSIGN_OR_THROW(
+ source, ::arrow::io::ReadableFile::Open(file,
reader_properties.memory_pool()));
+
+ auto file_reader = parquet::ParquetFileReader::Open(source,
reader_properties);
+ CheckPageIndex(file_reader.get(), file_decryption_properties);
+
+ ASSERT_NO_FATAL_FAILURE(file_reader->Close());
+ PARQUET_THROW_NOT_OK(source->Close());
+}
+
+void FileDecryptor::CheckPageIndex(
+ parquet::ParquetFileReader* file_reader,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::shared_ptr<PageIndexReader> page_index_reader =
file_reader->GetPageIndexReader();
+ ASSERT_NE(page_index_reader, nullptr);
+
+ const std::shared_ptr<parquet::FileMetaData> file_metadata =
file_reader->metadata();
+ const int num_row_groups = file_metadata->num_row_groups();
+ const int num_columns = file_metadata->num_columns();
+ ASSERT_EQ(num_columns, 8);
+
+ // We cannot read page index of encrypted columns in the plaintext mode
+ std::vector<int32_t> need_row_groups(num_row_groups);
+ std::for_each(need_row_groups.begin(), need_row_groups.end(),
+ [i = 0](int32_t& v) mutable { v = i++; });
+ std::vector<int32_t> need_columns;
+ if (file_decryption_properties == nullptr) {
+ need_columns = {0, 1, 2, 3, 6, 7};
+ } else {
+ need_columns = {0, 1, 2, 3, 4, 5, 6, 7};
+ }
+
+ // Provide hint of requested columns to avoid accessing encrypted columns
without
+ // decryption properties.
+ page_index_reader->WillNeed(
+ need_row_groups, need_columns,
+ PageIndexSelection{/*column_index=*/true, /*offset_index=*/true});
+
+ // Iterate over all the RowGroups in the file.
+ for (int r = 0; r < num_row_groups; ++r) {
+ auto row_group_page_index_reader = page_index_reader->RowGroup(r);
+ ASSERT_NE(row_group_page_index_reader, nullptr);
+
+ for (int c = 0; c < num_columns; ++c) {
+ // Skip reading encrypted columns without decryption properties.
+ if (file_decryption_properties == nullptr && (c == 4 || c == 5)) {
+ continue;
+ }
+
+ constexpr size_t kExpectedNumPages = 1;
+
+ // Check offset index.
+ auto offset_index = row_group_page_index_reader->GetOffsetIndex(c);
+ ASSERT_NE(offset_index, nullptr);
+ ASSERT_EQ(offset_index->page_locations().size(), kExpectedNumPages);
+ const auto& first_page = offset_index->page_locations()[0];
+ ASSERT_EQ(first_page.first_row_index, 0);
+ ASSERT_GT(first_page.compressed_page_size, 0);
+
+ // Int96 column does not have column index.
+ if (c == 3) {
+ continue;
+ }
+
+ // Check column index
+ auto column_index = row_group_page_index_reader->GetColumnIndex(c);
+ ASSERT_NE(column_index, nullptr);
+ ASSERT_EQ(column_index->null_pages().size(), kExpectedNumPages);
+ ASSERT_EQ(column_index->null_pages()[0], false);
+ ASSERT_EQ(column_index->encoded_min_values().size(), kExpectedNumPages);
+ ASSERT_EQ(column_index->encoded_max_values().size(), kExpectedNumPages);
+ ASSERT_TRUE(column_index->has_null_counts());
+
+ switch (c) {
+ case 0: {
+ auto bool_column_index =
+ std::dynamic_pointer_cast<BoolColumnIndex>(column_index);
+ ASSERT_NE(bool_column_index, nullptr);
+ constexpr int kExpectedNullCount = 0;
+ constexpr bool kExpectedMin = false;
+ constexpr bool kExpectedMax = true;
+ ASSERT_EQ(bool_column_index->null_counts().at(0),
kExpectedNullCount);
+ ASSERT_EQ(static_cast<bool>(bool_column_index->min_values()[0]),
kExpectedMin);
+ ASSERT_EQ(static_cast<bool>(bool_column_index->max_values()[0]),
kExpectedMax);
Review Comment:
I'm sure you can write a (templated?) helper function to avoid repeating
yourself here.
For example:
```c++
template <typename DType, typename c_type = typename DType::c_type>
void AssertColumnIndex(
const ColumnIndex&,
const std::vector<int64_t>& expected_null_counts,
const std::vector<c_type>& expected_min_values,
const std::vector<c_type>& expected_max_values);
```
##########
cpp/src/parquet/page_index.h:
##########
@@ -186,8 +191,7 @@ class PARQUET_EXPORT PageIndexReader {
/// that creates this PageIndexReader.
static std::shared_ptr<PageIndexReader> Make(
::arrow::io::RandomAccessFile* input, std::shared_ptr<FileMetaData>
file_metadata,
- const ReaderProperties& properties,
- std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
+ const ReaderProperties& properties, InternalFileDecryptor*
file_decryptor);
Review Comment:
Why do some APIs take a raw pointer and others a shared_ptr? Is there a
reason for the inconsistency?
##########
cpp/src/parquet/page_index.cc:
##########
@@ -830,14 +894,13 @@ RowGroupIndexReadRange
PageIndexReader::DeterminePageIndexRangesInRowGroup(
// ----------------------------------------------------------------------
// Public factory functions
-std::unique_ptr<ColumnIndex> ColumnIndex::Make(const ColumnDescriptor& descr,
- const void* serialized_index,
- uint32_t index_len,
- const ReaderProperties&
properties) {
+std::unique_ptr<ColumnIndex> ColumnIndex::Make(
+ const ColumnDescriptor& descr, const void* serialized_index, uint32_t
index_len,
+ const ReaderProperties& properties, const std::shared_ptr<Decryptor>&
decryptor) {
format::ColumnIndex column_index;
ThriftDeserializer deserializer(properties);
deserializer.DeserializeMessage(reinterpret_cast<const
uint8_t*>(serialized_index),
Review Comment:
Same for `SerializeMessage`, incidentally.
##########
cpp/src/parquet/page_index.cc:
##########
@@ -725,6 +761,29 @@ class PageIndexBuilderImpl final : public PageIndexBuilder
{
}
}
+ template <typename Builder>
Review Comment:
Instead of templating this on the builder type, why wouldn't you simply pass
the `module_type` as a function argument?
##########
cpp/src/parquet/page_index.cc:
##########
@@ -830,14 +894,13 @@ RowGroupIndexReadRange
PageIndexReader::DeterminePageIndexRangesInRowGroup(
// ----------------------------------------------------------------------
// Public factory functions
-std::unique_ptr<ColumnIndex> ColumnIndex::Make(const ColumnDescriptor& descr,
- const void* serialized_index,
- uint32_t index_len,
- const ReaderProperties&
properties) {
+std::unique_ptr<ColumnIndex> ColumnIndex::Make(
+ const ColumnDescriptor& descr, const void* serialized_index, uint32_t
index_len,
+ const ReaderProperties& properties, const std::shared_ptr<Decryptor>&
decryptor) {
format::ColumnIndex column_index;
ThriftDeserializer deserializer(properties);
deserializer.DeserializeMessage(reinterpret_cast<const
uint8_t*>(serialized_index),
Review Comment:
Can `DeserializeMessage` be updated to take a raw `Decryptor*`? It doesn't
need to take ownership...
##########
cpp/src/parquet/encryption/test_encryption_util.cc:
##########
@@ -509,4 +513,178 @@ void FileDecryptor::CheckFile(parquet::ParquetFileReader*
file_reader,
}
}
+void FileDecryptor::DecryptPageIndex(
+ const std::string& file,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::string exception_msg;
+ parquet::ReaderProperties reader_properties =
parquet::default_reader_properties();
+ if (file_decryption_properties) {
+
reader_properties.file_decryption_properties(file_decryption_properties->DeepClone());
+ }
+
+ std::shared_ptr<::arrow::io::RandomAccessFile> source;
+ PARQUET_ASSIGN_OR_THROW(
+ source, ::arrow::io::ReadableFile::Open(file,
reader_properties.memory_pool()));
+
+ auto file_reader = parquet::ParquetFileReader::Open(source,
reader_properties);
+ CheckPageIndex(file_reader.get(), file_decryption_properties);
+
+ ASSERT_NO_FATAL_FAILURE(file_reader->Close());
+ PARQUET_THROW_NOT_OK(source->Close());
+}
+
+void FileDecryptor::CheckPageIndex(
+ parquet::ParquetFileReader* file_reader,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::shared_ptr<PageIndexReader> page_index_reader =
file_reader->GetPageIndexReader();
+ ASSERT_NE(page_index_reader, nullptr);
+
+ const std::shared_ptr<parquet::FileMetaData> file_metadata =
file_reader->metadata();
+ const int num_row_groups = file_metadata->num_row_groups();
+ const int num_columns = file_metadata->num_columns();
+ ASSERT_EQ(num_columns, 8);
+
+ // We cannot read page index of encrypted columns in the plaintext mode
+ std::vector<int32_t> need_row_groups(num_row_groups);
+ std::for_each(need_row_groups.begin(), need_row_groups.end(),
+ [i = 0](int32_t& v) mutable { v = i++; });
Review Comment:
Aren't you reimplementing `std::iota` here?
##########
cpp/src/parquet/page_index.h:
##########
@@ -332,7 +340,8 @@ class PARQUET_EXPORT OffsetIndexBuilder {
class PARQUET_EXPORT PageIndexBuilder {
public:
/// \brief API convenience to create a PageIndexBuilder.
- static std::unique_ptr<PageIndexBuilder> Make(const SchemaDescriptor*
schema);
+ static std::unique_ptr<PageIndexBuilder> Make(const SchemaDescriptor* schema,
+ InternalFileEncryptor*
file_encryptor);
Review Comment:
These APIs are probably not widely-used, but they are still public and we
should minimize API compatibility breakage. Can we make the
encryption/decryption arguments optional? (or have overloads without them)
##########
cpp/src/parquet/encryption/test_encryption_util.cc:
##########
@@ -509,4 +513,178 @@ void FileDecryptor::CheckFile(parquet::ParquetFileReader*
file_reader,
}
}
+void FileDecryptor::DecryptPageIndex(
+ const std::string& file,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::string exception_msg;
+ parquet::ReaderProperties reader_properties =
parquet::default_reader_properties();
+ if (file_decryption_properties) {
+
reader_properties.file_decryption_properties(file_decryption_properties->DeepClone());
+ }
+
+ std::shared_ptr<::arrow::io::RandomAccessFile> source;
+ PARQUET_ASSIGN_OR_THROW(
+ source, ::arrow::io::ReadableFile::Open(file,
reader_properties.memory_pool()));
+
+ auto file_reader = parquet::ParquetFileReader::Open(source,
reader_properties);
+ CheckPageIndex(file_reader.get(), file_decryption_properties);
+
+ ASSERT_NO_FATAL_FAILURE(file_reader->Close());
+ PARQUET_THROW_NOT_OK(source->Close());
+}
+
+void FileDecryptor::CheckPageIndex(
+ parquet::ParquetFileReader* file_reader,
+ const std::shared_ptr<FileDecryptionProperties>&
file_decryption_properties) {
+ std::shared_ptr<PageIndexReader> page_index_reader =
file_reader->GetPageIndexReader();
+ ASSERT_NE(page_index_reader, nullptr);
+
+ const std::shared_ptr<parquet::FileMetaData> file_metadata =
file_reader->metadata();
+ const int num_row_groups = file_metadata->num_row_groups();
+ const int num_columns = file_metadata->num_columns();
+ ASSERT_EQ(num_columns, 8);
+
+ // We cannot read page index of encrypted columns in the plaintext mode
+ std::vector<int32_t> need_row_groups(num_row_groups);
+ std::for_each(need_row_groups.begin(), need_row_groups.end(),
+ [i = 0](int32_t& v) mutable { v = i++; });
+ std::vector<int32_t> need_columns;
+ if (file_decryption_properties == nullptr) {
+ need_columns = {0, 1, 2, 3, 6, 7};
+ } else {
+ need_columns = {0, 1, 2, 3, 4, 5, 6, 7};
+ }
+
+ // Provide hint of requested columns to avoid accessing encrypted columns
without
+ // decryption properties.
+ page_index_reader->WillNeed(
+ need_row_groups, need_columns,
+ PageIndexSelection{/*column_index=*/true, /*offset_index=*/true});
+
+ // Iterate over all the RowGroups in the file.
+ for (int r = 0; r < num_row_groups; ++r) {
+ auto row_group_page_index_reader = page_index_reader->RowGroup(r);
+ ASSERT_NE(row_group_page_index_reader, nullptr);
+
+ for (int c = 0; c < num_columns; ++c) {
+ // Skip reading encrypted columns without decryption properties.
Review Comment:
Instead of skipping, shouldn't we check that reading said columns fails?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
