adamreeve commented on code in PR #9203:
URL: https://github.com/apache/arrow-rs/pull/9203#discussion_r3135112017
##########
parquet/tests/encryption/encryption.rs:
##########
@@ -43,119 +44,243 @@ use std::sync::Arc;
#[test]
fn test_non_uniform_encryption_plaintext_footer() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_plaintext_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn non_uniform_encryption_plaintext_footer(footer_key: &[u8], column_keys:
&[(&str, &[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_plaintext_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- // There is always a footer key even with a plaintext footer,
+ // AES-128: there is always a footer key even with a plaintext footer,
// but this is used for signing the footer.
- let footer_key = "0123456789012345".as_bytes(); // 128bit/16
- let column_1_key = "1234567890123450".as_bytes();
- let column_2_key = "1234567890123451".as_bytes();
-
- let decryption_properties =
FileDecryptionProperties::builder(footer_key.to_vec())
- .with_column_key("double_field", column_1_key.to_vec())
- .with_column_key("float_field", column_2_key.to_vec())
- .build()
- .unwrap();
+ non_uniform_encryption_plaintext_footer(
+ b"0123456789012345", // 128bit/16
Review Comment:
Can we pull these repeated footer keys and column key arrays out as
constants and put them in the `encryption_util` module to reduce the
duplication?
##########
parquet/tests/encryption/encryption.rs:
##########
@@ -167,219 +292,470 @@ fn test_plaintext_footer_read_without_decryption() {
#[test]
fn test_non_uniform_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_and_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn non_uniform_encryption(footer_key: &[u8], column_keys: &[(&str,
&[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_and_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let footer_key = b"0123456789012345".to_vec(); // 128bit/16
- let column_1_key = b"1234567890123450".to_vec();
- let column_2_key = b"1234567890123451".to_vec();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
- let decryption_properties = FileDecryptionProperties::builder(footer_key)
- .with_column_key("double_field", column_1_key)
- .with_column_key("float_field", column_2_key)
- .build()
- .unwrap();
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-128
+ non_uniform_encryption(
+ b"0123456789012345",
+ &[
+ ("double_field", b"1234567890123450".as_slice()),
+ ("float_field", b"1234567890123451".as_slice()),
+ ],
+ );
+
+ // AES-256
+ non_uniform_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[
+ (
+ "double_field",
+ b"12345678901234567890123456789012".as_slice(),
+ ),
+ (
+ "float_field",
+ b"12345678901234567890123456789013".as_slice(),
+ ),
+ (
+ "boolean_field",
+ b"12345678901234567890123456789014".as_slice(),
+ ),
+ (
+ "int32_field",
+ b"12345678901234567890123456789015".as_slice(),
+ ),
+ ("ba_field", b"12345678901234567890123456789016".as_slice()),
+ ("flba_field", b"12345678901234567890123456789017".as_slice()),
+ (
+ "int64_field",
+ b"12345678901234567890123456789018".as_slice(),
+ ),
+ (
+ "int96_field",
+ b"12345678901234567890123456789019".as_slice(),
+ ),
+ ],
+ );
}
#[test]
fn test_uniform_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/uniform_encryption.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn uniform_encryption(footer_key: &[u8], column_keys: &[(&str, &[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "uniform_encryption.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let key_code = b"0123456789012345".to_vec();
- let decryption_properties =
FileDecryptionProperties::builder(key_code).build().unwrap();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
+
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-128: there is always a footer key even with a plaintext footer,
+ // but this is used for signing the footer.
Review Comment:
I don't think this comment is relevant here, this test uses an encrypted
footer.
##########
parquet/tests/encryption/encryption.rs:
##########
@@ -167,219 +292,470 @@ fn test_plaintext_footer_read_without_decryption() {
#[test]
fn test_non_uniform_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_and_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn non_uniform_encryption(footer_key: &[u8], column_keys: &[(&str,
&[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_and_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let footer_key = b"0123456789012345".to_vec(); // 128bit/16
- let column_1_key = b"1234567890123450".to_vec();
- let column_2_key = b"1234567890123451".to_vec();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
- let decryption_properties = FileDecryptionProperties::builder(footer_key)
- .with_column_key("double_field", column_1_key)
- .with_column_key("float_field", column_2_key)
- .build()
- .unwrap();
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-128
+ non_uniform_encryption(
+ b"0123456789012345",
+ &[
+ ("double_field", b"1234567890123450".as_slice()),
+ ("float_field", b"1234567890123451".as_slice()),
+ ],
+ );
+
+ // AES-256
+ non_uniform_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[
+ (
+ "double_field",
+ b"12345678901234567890123456789012".as_slice(),
+ ),
+ (
+ "float_field",
+ b"12345678901234567890123456789013".as_slice(),
+ ),
+ (
+ "boolean_field",
+ b"12345678901234567890123456789014".as_slice(),
+ ),
+ (
+ "int32_field",
+ b"12345678901234567890123456789015".as_slice(),
+ ),
+ ("ba_field", b"12345678901234567890123456789016".as_slice()),
+ ("flba_field", b"12345678901234567890123456789017".as_slice()),
+ (
+ "int64_field",
+ b"12345678901234567890123456789018".as_slice(),
+ ),
+ (
+ "int96_field",
+ b"12345678901234567890123456789019".as_slice(),
+ ),
+ ],
+ );
}
#[test]
fn test_uniform_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/uniform_encryption.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn uniform_encryption(footer_key: &[u8], column_keys: &[(&str, &[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "uniform_encryption.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let key_code = b"0123456789012345".to_vec();
- let decryption_properties =
FileDecryptionProperties::builder(key_code).build().unwrap();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
+
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-128: there is always a footer key even with a plaintext footer,
+ // but this is used for signing the footer.
+ uniform_encryption(
+ b"0123456789012345", // 128bit/16
+ &[],
+ );
+
+ // AES-256
+ uniform_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[],
+ );
}
#[test]
fn test_decrypting_without_decryption_properties_fails() {
let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/uniform_encryption.parquet.encrypted");
- let file = File::open(path).unwrap();
+ let paths = [
+ format!("{test_data}/uniform_encryption.parquet.encrypted"),
+ format!("{test_data}/aes256/uniform_encryption.parquet.encrypted"),
+ ];
- let options = ArrowReaderOptions::default();
- let result = ArrowReaderMetadata::load(&file, options.clone());
- assert!(result.is_err());
- assert_eq!(
- result.unwrap_err().to_string(),
- "Parquet error: Parquet file has an encrypted footer but decryption
properties were not provided"
- );
+ for path in &paths {
+ let file = File::open(path).unwrap();
+
+ let options = ArrowReaderOptions::default();
+ let result = ArrowReaderMetadata::load(&file, options.clone());
+ assert!(result.is_err());
+ assert_eq!(
+ result.unwrap_err().to_string(),
+ "Parquet error: Parquet file has an encrypted footer but
decryption properties were not provided"
+ );
+ }
}
#[test]
fn test_aes_ctr_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_and_footer_ctr.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn aes_ctr_encryption(footer_key: &[u8], column_keys: &[(&str, &[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_and_footer_ctr.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let footer_key = b"0123456789012345".to_vec();
- let column_1_key = b"1234567890123450".to_vec();
- let column_2_key = b"1234567890123451".to_vec();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
- let decryption_properties = FileDecryptionProperties::builder(footer_key)
- .with_column_key("double_field", column_1_key)
- .with_column_key("float_field", column_2_key)
- .build()
- .unwrap();
+ let options =
+
ArrowReaderOptions::new().with_file_decryption_properties(decryption_properties);
+ let metadata = ArrowReaderMetadata::load(&file, options);
- let options =
-
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
- let metadata = ArrowReaderMetadata::load(&file, options);
+ match metadata {
+ Err(ParquetError::NYI(s)) => {
+ assert!(s.contains("AES_GCM_CTR_V1"));
+ }
+ _ => {
+ panic!("Expected ParquetError::NYI");
+ }
+ };
+ }
+ // AES-128
+ aes_ctr_encryption(
+ b"0123456789012345", // 128bit/16
+ &[
+ ("double_field", b"1234567890123450".as_slice()),
+ ("float_field", b"1234567890123451".as_slice()),
+ ],
+ );
- match metadata {
- Err(parquet::errors::ParquetError::NYI(s)) => {
- assert!(s.contains("AES_GCM_CTR_V1"));
- }
- _ => {
- panic!("Expected ParquetError::NYI");
- }
- };
+ // AES-256
+ aes_ctr_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[
+ (
+ "double_field",
+ b"12345678901234567890123456789012".as_slice(),
+ ),
+ (
+ "float_field",
+ b"12345678901234567890123456789013".as_slice(),
+ ),
+ ],
+ );
}
#[test]
fn test_non_uniform_encryption_plaintext_footer_with_key_retriever() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_plaintext_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn non_uniform_encryption_plaintext_footer_with_key_retriever(
+ footer_key: &[u8],
+ keys: &[(&str, &[u8])],
+ ) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_plaintext_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let key_retriever = TestKeyRetriever::new()
- .with_key("kf".to_owned(), "0123456789012345".as_bytes().to_vec())
- .with_key("kc1".to_owned(), "1234567890123450".as_bytes().to_vec())
- .with_key("kc2".to_owned(), "1234567890123451".as_bytes().to_vec());
+ let mut key_retriever = TestKeyRetriever::new();
+ for (key_name, key) in keys {
+ key_retriever = key_retriever.with_key((*key_name).to_owned(),
(*key).to_vec());
+ }
- let decryption_properties =
- FileDecryptionProperties::with_key_retriever(Arc::new(key_retriever))
- .build()
- .unwrap();
+ let decryption_properties =
+
FileDecryptionProperties::with_key_retriever(Arc::new(key_retriever))
+ .build()
+ .unwrap();
+
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
+
+ // AES-128
+ non_uniform_encryption_plaintext_footer_with_key_retriever(
+ b"0123456789012345",
+ &[
+ ("kf", b"0123456789012345".as_slice()),
+ ("kc1", b"1234567890123450".as_slice()),
+ ("kc2", b"1234567890123451".as_slice()),
+ ],
+ );
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-256
+ non_uniform_encryption_plaintext_footer_with_key_retriever(
+ b"01234567890123456789012345678901",
+ &[
+ ("kf", b"01234567890123456789012345678901".as_slice()),
+ ("kc1", b"12345678901234567890123456789012".as_slice()),
+ ("kc2", b"12345678901234567890123456789013".as_slice()),
+ ("kc3", b"12345678901234567890123456789014".as_slice()),
+ ("kc4", b"12345678901234567890123456789015".as_slice()),
+ ("kc5", b"12345678901234567890123456789016".as_slice()),
+ ("kc6", b"12345678901234567890123456789017".as_slice()),
+ ("kc7", b"12345678901234567890123456789018".as_slice()),
+ ("kc8", b"12345678901234567890123456789019".as_slice()),
+ ],
+ );
}
#[test]
fn test_uniform_encryption_plaintext_footer_with_key_retriever() {
- let test_data = arrow::util::test_util::parquet_test_data();
+ fn uniform_encryption_plaintext_footer_with_key_retriever(
+ footer_key: &[u8],
+ footer_key_metadata: &str,
+ wrong_footer_key: &[u8],
+ dec_column_keys: &[(&str, &[u8])],
+ enc_column_keys: &[(&str, &[u8], &str)],
+ ) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_plaintext_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- // Read example data with key retriever
- let path =
format!("{test_data}/encrypt_columns_plaintext_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ let mut key_retriever =
+ TestKeyRetriever::new().with_key(footer_key_metadata.to_owned(),
footer_key.to_vec());
- let key_retriever = Arc::new(
- TestKeyRetriever::new()
- .with_key("kf".to_owned(), b"0123456789012345".to_vec())
- .with_key("kc1".to_owned(), b"1234567890123450".to_vec())
- .with_key("kc2".to_owned(), b"1234567890123451".to_vec()),
- );
+ for (key_name, key) in dec_column_keys {
+ key_retriever = key_retriever.with_key((*key_name).to_owned(),
(*key).to_vec());
+ }
- let decryption_properties =
FileDecryptionProperties::with_key_retriever(key_retriever.clone())
- .build()
- .unwrap();
+ let decryption_properties =
+
FileDecryptionProperties::with_key_retriever(Arc::new(key_retriever))
+ .build()
+ .unwrap();
- let options =
-
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
- let metadata = ArrowReaderMetadata::load(&file, options.clone()).unwrap();
+ let options = ArrowReaderOptions::default()
+ .with_file_decryption_properties(decryption_properties.clone());
+ let metadata = ArrowReaderMetadata::load(&file,
options.clone()).unwrap();
+
+ // Write data into temporary file with plaintext footer and footer key
metadata
+ let temp_file = tempfile::tempfile().unwrap();
+ let mut encryption_properties_builder =
+ FileEncryptionProperties::builder(footer_key.to_vec())
+ .with_footer_key_metadata(footer_key_metadata.into());
+ for (column_name, key, metadata) in enc_column_keys {
+ encryption_properties_builder = encryption_properties_builder
+ .with_column_key_and_metadata(column_name, key.to_vec(),
(*metadata).into());
+ }
+ let encryption_properties = encryption_properties_builder
+ .with_plaintext_footer(true)
+ .build()
+ .unwrap();
- // Write data into temporary file with plaintext footer and footer key
metadata
- let temp_file = tempfile::tempfile().unwrap();
- let encryption_properties =
FileEncryptionProperties::builder(b"0123456789012345".to_vec())
- .with_footer_key_metadata("kf".into())
- .with_column_key_and_metadata("double_field",
b"1234567890123450".to_vec(), b"kc1".into())
- .with_column_key_and_metadata("float_field",
b"1234567890123451".to_vec(), b"kc2".into())
- .with_plaintext_footer(true)
- .build()
- .unwrap();
+ let builder =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap();
+ let batch_reader = builder.build().unwrap();
+ let batches = batch_reader
+ .collect::<parquet::errors::Result<Vec<RecordBatch>, _>>()
+ .unwrap();
- let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file,
options).unwrap();
- let batch_reader = builder.build().unwrap();
- let batches = batch_reader
- .collect::<parquet::errors::Result<Vec<RecordBatch>, _>>()
+ let props = WriterProperties::builder()
+ .with_file_encryption_properties(encryption_properties)
+ .build();
+
+ let mut writer = ArrowWriter::try_new(
+ temp_file.try_clone().unwrap(),
+ metadata.schema().clone(),
+ Some(props),
+ )
.unwrap();
+ for batch in batches {
+ writer.write(&batch).unwrap();
+ }
- let props = WriterProperties::builder()
- .with_file_encryption_properties(encryption_properties)
- .build();
+ writer.close().unwrap();
- let mut writer = ArrowWriter::try_new(
- temp_file.try_clone().unwrap(),
- metadata.schema().clone(),
- Some(props),
- )
- .unwrap();
- for batch in batches {
- writer.write(&batch).unwrap();
- }
+ // Read temporary file with plaintext metadata using key retriever
+ let options = ArrowReaderOptions::default()
+ .with_file_decryption_properties(decryption_properties.clone());
+ let _ = ArrowReaderMetadata::load(&temp_file,
options.clone()).unwrap();
- writer.close().unwrap();
+ // Read temporary file with plaintext metadata using key retriever
with invalid key
+ let mut key_retriever = TestKeyRetriever::new()
+ .with_key(footer_key_metadata.to_owned(),
wrong_footer_key.to_vec());
- // Read temporary file with plaintext metadata using key retriever
- let decryption_properties =
FileDecryptionProperties::with_key_retriever(key_retriever)
- .build()
- .unwrap();
+ for (key_name, key) in dec_column_keys {
+ key_retriever = key_retriever.with_key((*key_name).to_owned(),
(*key).to_vec());
+ }
+ let decryption_properties =
+
FileDecryptionProperties::with_key_retriever(Arc::new(key_retriever))
+ .build()
+ .unwrap();
+ let options =
+
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
+ let result = ArrowReaderMetadata::load(&temp_file, options.clone());
+ assert!(result.is_err());
+ assert!(
+ result
+ .unwrap_err()
+ .to_string()
+ .starts_with("Parquet error: Footer signature verification
failed. Computed: [")
+ );
+ }
- let options =
-
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
- let _ = ArrowReaderMetadata::load(&temp_file, options.clone()).unwrap();
-
- // Read temporary file with plaintext metadata using key retriever with
invalid key
- let key_retriever = Arc::new(
- TestKeyRetriever::new()
- .with_key("kf".to_owned(), b"0133756789012345".to_vec())
- .with_key("kc1".to_owned(), b"1234567890123450".to_vec())
- .with_key("kc2".to_owned(), b"1234567890123451".to_vec()),
+ // AES-128
+ uniform_encryption_plaintext_footer_with_key_retriever(
+ b"0123456789012345",
+ "kf",
+ b"0133456789012345",
+ &[
+ ("kc1", b"1234567890123450".as_slice()),
+ ("kc2", b"1234567890123451".as_slice()),
+ ],
+ &[
+ ("double_field", b"1234567890123450".as_slice(), "kc1"),
+ ("float_field", b"1234567890123451".as_slice(), "kc2"),
+ ],
);
- let decryption_properties =
FileDecryptionProperties::with_key_retriever(key_retriever)
- .build()
- .unwrap();
- let options =
-
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
- let result = ArrowReaderMetadata::load(&temp_file, options.clone());
- assert!(result.is_err());
- assert!(
- result
- .unwrap_err()
- .to_string()
- .starts_with("Parquet error: Footer signature verification failed.
Computed: [")
+
+ // AES-256
+ // The asymmetric column names is because we check column paths in the
validate_encrypted_column_names function of [encryption::encrypt]
+ // The column path of the repeated field (int64_field) is
int64_field.list.int64_field
+ // Switching from `c.path().string()` to `c.name().to_string()` can fix
the asymmetricity
Review Comment:
This comment doesn't seem relevant here. Because this uses a key retriever,
we don't need to provide column names to configure decryption.
##########
parquet/tests/encryption/encryption.rs:
##########
@@ -167,219 +292,470 @@ fn test_plaintext_footer_read_without_decryption() {
#[test]
fn test_non_uniform_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_and_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn non_uniform_encryption(footer_key: &[u8], column_keys: &[(&str,
&[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_and_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let footer_key = b"0123456789012345".to_vec(); // 128bit/16
- let column_1_key = b"1234567890123450".to_vec();
- let column_2_key = b"1234567890123451".to_vec();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
- let decryption_properties = FileDecryptionProperties::builder(footer_key)
- .with_column_key("double_field", column_1_key)
- .with_column_key("float_field", column_2_key)
- .build()
- .unwrap();
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-128
+ non_uniform_encryption(
+ b"0123456789012345",
+ &[
+ ("double_field", b"1234567890123450".as_slice()),
+ ("float_field", b"1234567890123451".as_slice()),
+ ],
+ );
+
+ // AES-256
+ non_uniform_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[
+ (
+ "double_field",
+ b"12345678901234567890123456789012".as_slice(),
+ ),
+ (
+ "float_field",
+ b"12345678901234567890123456789013".as_slice(),
+ ),
+ (
+ "boolean_field",
+ b"12345678901234567890123456789014".as_slice(),
+ ),
+ (
+ "int32_field",
+ b"12345678901234567890123456789015".as_slice(),
+ ),
+ ("ba_field", b"12345678901234567890123456789016".as_slice()),
+ ("flba_field", b"12345678901234567890123456789017".as_slice()),
+ (
+ "int64_field",
+ b"12345678901234567890123456789018".as_slice(),
+ ),
+ (
+ "int96_field",
+ b"12345678901234567890123456789019".as_slice(),
+ ),
+ ],
+ );
}
#[test]
fn test_uniform_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/uniform_encryption.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn uniform_encryption(footer_key: &[u8], column_keys: &[(&str, &[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "uniform_encryption.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let key_code = b"0123456789012345".to_vec();
- let decryption_properties =
FileDecryptionProperties::builder(key_code).build().unwrap();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
+
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-128: there is always a footer key even with a plaintext footer,
+ // but this is used for signing the footer.
+ uniform_encryption(
+ b"0123456789012345", // 128bit/16
+ &[],
+ );
+
+ // AES-256
+ uniform_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[],
+ );
}
#[test]
fn test_decrypting_without_decryption_properties_fails() {
let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/uniform_encryption.parquet.encrypted");
- let file = File::open(path).unwrap();
+ let paths = [
+ format!("{test_data}/uniform_encryption.parquet.encrypted"),
+ format!("{test_data}/aes256/uniform_encryption.parquet.encrypted"),
+ ];
- let options = ArrowReaderOptions::default();
- let result = ArrowReaderMetadata::load(&file, options.clone());
- assert!(result.is_err());
- assert_eq!(
- result.unwrap_err().to_string(),
- "Parquet error: Parquet file has an encrypted footer but decryption
properties were not provided"
- );
+ for path in &paths {
+ let file = File::open(path).unwrap();
+
+ let options = ArrowReaderOptions::default();
+ let result = ArrowReaderMetadata::load(&file, options.clone());
+ assert!(result.is_err());
+ assert_eq!(
+ result.unwrap_err().to_string(),
+ "Parquet error: Parquet file has an encrypted footer but
decryption properties were not provided"
+ );
+ }
}
#[test]
fn test_aes_ctr_encryption() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_and_footer_ctr.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn aes_ctr_encryption(footer_key: &[u8], column_keys: &[(&str, &[u8])]) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_and_footer_ctr.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let footer_key = b"0123456789012345".to_vec();
- let column_1_key = b"1234567890123450".to_vec();
- let column_2_key = b"1234567890123451".to_vec();
+ let mut builder =
FileDecryptionProperties::builder(footer_key.to_vec());
+ for (column_name, key) in column_keys {
+ builder = builder.with_column_key(column_name, key.to_vec());
+ }
+ let decryption_properties = builder.build().unwrap();
- let decryption_properties = FileDecryptionProperties::builder(footer_key)
- .with_column_key("double_field", column_1_key)
- .with_column_key("float_field", column_2_key)
- .build()
- .unwrap();
+ let options =
+
ArrowReaderOptions::new().with_file_decryption_properties(decryption_properties);
+ let metadata = ArrowReaderMetadata::load(&file, options);
- let options =
-
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
- let metadata = ArrowReaderMetadata::load(&file, options);
+ match metadata {
+ Err(ParquetError::NYI(s)) => {
+ assert!(s.contains("AES_GCM_CTR_V1"));
+ }
+ _ => {
+ panic!("Expected ParquetError::NYI");
+ }
+ };
+ }
+ // AES-128
+ aes_ctr_encryption(
+ b"0123456789012345", // 128bit/16
+ &[
+ ("double_field", b"1234567890123450".as_slice()),
+ ("float_field", b"1234567890123451".as_slice()),
+ ],
+ );
- match metadata {
- Err(parquet::errors::ParquetError::NYI(s)) => {
- assert!(s.contains("AES_GCM_CTR_V1"));
- }
- _ => {
- panic!("Expected ParquetError::NYI");
- }
- };
+ // AES-256
+ aes_ctr_encryption(
+ b"01234567890123456789012345678901", // 256bit/32
+ &[
+ (
+ "double_field",
+ b"12345678901234567890123456789012".as_slice(),
+ ),
+ (
+ "float_field",
+ b"12345678901234567890123456789013".as_slice(),
+ ),
+ ],
+ );
}
#[test]
fn test_non_uniform_encryption_plaintext_footer_with_key_retriever() {
- let test_data = arrow::util::test_util::parquet_test_data();
- let path =
format!("{test_data}/encrypt_columns_plaintext_footer.parquet.encrypted");
- let file = File::open(path).unwrap();
+ fn non_uniform_encryption_plaintext_footer_with_key_retriever(
+ footer_key: &[u8],
+ keys: &[(&str, &[u8])],
+ ) {
+ let path = encryption_util::encrypted_data_path(
+ footer_key,
+ "encrypt_columns_plaintext_footer.parquet.encrypted",
+ );
+ let file = File::open(path).unwrap();
- let key_retriever = TestKeyRetriever::new()
- .with_key("kf".to_owned(), "0123456789012345".as_bytes().to_vec())
- .with_key("kc1".to_owned(), "1234567890123450".as_bytes().to_vec())
- .with_key("kc2".to_owned(), "1234567890123451".as_bytes().to_vec());
+ let mut key_retriever = TestKeyRetriever::new();
+ for (key_name, key) in keys {
+ key_retriever = key_retriever.with_key((*key_name).to_owned(),
(*key).to_vec());
+ }
- let decryption_properties =
- FileDecryptionProperties::with_key_retriever(Arc::new(key_retriever))
- .build()
- .unwrap();
+ let decryption_properties =
+
FileDecryptionProperties::with_key_retriever(Arc::new(key_retriever))
+ .build()
+ .unwrap();
+
+ verify_encryption_test_file_read(file, decryption_properties);
+ }
+
+ // AES-128
+ non_uniform_encryption_plaintext_footer_with_key_retriever(
+ b"0123456789012345",
+ &[
+ ("kf", b"0123456789012345".as_slice()),
+ ("kc1", b"1234567890123450".as_slice()),
+ ("kc2", b"1234567890123451".as_slice()),
+ ],
+ );
- verify_encryption_test_file_read(file, decryption_properties);
+ // AES-256
+ non_uniform_encryption_plaintext_footer_with_key_retriever(
+ b"01234567890123456789012345678901",
+ &[
+ ("kf", b"01234567890123456789012345678901".as_slice()),
+ ("kc1", b"12345678901234567890123456789012".as_slice()),
+ ("kc2", b"12345678901234567890123456789013".as_slice()),
+ ("kc3", b"12345678901234567890123456789014".as_slice()),
+ ("kc4", b"12345678901234567890123456789015".as_slice()),
+ ("kc5", b"12345678901234567890123456789016".as_slice()),
+ ("kc6", b"12345678901234567890123456789017".as_slice()),
+ ("kc7", b"12345678901234567890123456789018".as_slice()),
+ ("kc8", b"12345678901234567890123456789019".as_slice()),
+ ],
+ );
}
#[test]
fn test_uniform_encryption_plaintext_footer_with_key_retriever() {
- let test_data = arrow::util::test_util::parquet_test_data();
+ fn uniform_encryption_plaintext_footer_with_key_retriever(
Review Comment:
This is a pre-existing problem, but this test says it's testing
"uniform_encryption", but actually specifies column encryption keys. There's a
`test_non_uniform_encryption_plaintext_footer_with_key_retriever` test above
already. It looks like the difference is this one rewrites the file. Maybe this
should be called
`test_roundtrip_non_uniform_encryption_plaintext_footer_with_key_retriever`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
