This is an automated email from the ASF dual-hosted git repository.
gershinsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new efb7554fa complete column encr (#1178)
efb7554fa is described below
commit efb7554faa1e1d7649916fa175430e21a266153e
Author: ggershinsky <[email protected]>
AuthorDate: Thu Oct 26 14:40:14 2023 +0300
complete column encr (#1178)
---
.../parquet/crypto/FileEncryptionProperties.java | 37 +++++++++++++++++----
.../parquet/hadoop/TestEncryptionOptions.java | 38 +++++++++++++++++++---
2 files changed, 63 insertions(+), 12 deletions(-)
diff --git
a/parquet-hadoop/src/main/java/org/apache/parquet/crypto/FileEncryptionProperties.java
b/parquet-hadoop/src/main/java/org/apache/parquet/crypto/FileEncryptionProperties.java
index 96a570a77..208d53e7a 100755
---
a/parquet-hadoop/src/main/java/org/apache/parquet/crypto/FileEncryptionProperties.java
+++
b/parquet-hadoop/src/main/java/org/apache/parquet/crypto/FileEncryptionProperties.java
@@ -33,6 +33,7 @@ public class FileEncryptionProperties {
private static final ParquetCipher ALGORITHM_DEFAULT =
ParquetCipher.AES_GCM_V1;
private static final boolean ENCRYPTED_FOOTER_DEFAULT = true;
+ private static final boolean COMPLETE_COLUMN_ENCRYPTION_DEFAULT = false;
private final EncryptionAlgorithm algorithm;
private final boolean encryptedFooter;
@@ -40,11 +41,13 @@ public class FileEncryptionProperties {
private final byte[] footerKeyMetadata;
private final byte[] fileAAD;
private final Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap;
+ private final boolean completeColumnEncryption;
private FileEncryptionProperties(ParquetCipher cipher,
byte[] footerKey, byte[] footerKeyMetadata, boolean encryptedFooter,
byte[] aadPrefix, boolean storeAadPrefixInFile,
- Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap) {
+ Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap,
+ boolean completeColumnEncryption) {
if (null == footerKey) {
throw new IllegalArgumentException("Footer key is null");
@@ -52,8 +55,14 @@ public class FileEncryptionProperties {
if (! (footerKey.length == 16 || footerKey.length == 24 ||
footerKey.length == 32)) {
throw new IllegalArgumentException("Wrong footer key length " +
footerKey.length);
}
- if (null != columnPropertyMap && columnPropertyMap.size() == 0) {
- throw new IllegalArgumentException("No encrypted columns");
+ if (null != columnPropertyMap) {
+ if (columnPropertyMap.size() == 0) {
+ throw new IllegalArgumentException("No encrypted columns");
+ }
+ } else {
+ if (completeColumnEncryption) {
+ throw new IllegalArgumentException("Encrypted columns are not
specified, cannot complete");
+ }
}
SecureRandom random = new SecureRandom();
@@ -88,6 +97,7 @@ public class FileEncryptionProperties {
this.footerKeyMetadata = footerKeyMetadata;
this.encryptedFooter = encryptedFooter;
this.columnPropertyMap = columnPropertyMap;
+ this.completeColumnEncryption = completeColumnEncryption;
}
/**
@@ -109,10 +119,12 @@ public class FileEncryptionProperties {
private byte[] aadPrefix;
private Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap;
private boolean storeAadPrefixInFile;
+ private boolean completeColumnEncryption;
private Builder(byte[] footerKey) {
this.parquetCipher = ALGORITHM_DEFAULT;
this.encryptedFooter = ENCRYPTED_FOOTER_DEFAULT;
+ this.completeColumnEncryption = COMPLETE_COLUMN_ENCRYPTION_DEFAULT;
this.footerKeyBytes = new byte[footerKey.length];
System.arraycopy(footerKey, 0, this.footerKeyBytes, 0, footerKey.length);
}
@@ -229,11 +241,17 @@ public class FileEncryptionProperties {
return this;
}
+ public Builder withCompleteColumnEncryption() {
+ this.completeColumnEncryption = true;
+
+ return this;
+ }
+
public FileEncryptionProperties build() {
return new FileEncryptionProperties(parquetCipher,
footerKeyBytes, footerKeyMetadata, encryptedFooter,
aadPrefix, storeAadPrefixInFile,
- columnPropertyMap);
+ columnPropertyMap, completeColumnEncryption);
}
}
@@ -261,9 +279,14 @@ public class FileEncryptionProperties {
ColumnEncryptionProperties columnProperties =
columnPropertyMap.get(columnPath);
if (null != columnProperties) {
return columnProperties;
- } else {
- // plaintext column
- return ColumnEncryptionProperties.builder(columnPath, false).build();
+ } else { // not set explicitly
+ if (completeColumnEncryption) {
+ // encrypted with footer key
+ return ColumnEncryptionProperties.builder(columnPath, true).build();
+ } else {
+ // plaintext column
+ return ColumnEncryptionProperties.builder(columnPath, false).build();
+ }
}
}
}
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
index a212c091f..e3ddb3e4c 100644
---
a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
@@ -20,7 +20,6 @@ package org.apache.parquet.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.parquet.crypto.ColumnDecryptionProperties;
@@ -45,11 +44,8 @@ import org.junit.rules.ErrorCollector;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import okhttp3.ConnectionSpec;
-import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
-import okhttp3.RequestBody;
import okhttp3.Response;
import java.io.IOException;
@@ -100,6 +96,9 @@ import static
org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
* ENCRYPT_COLUMNS_PLAINTEXT_FOOTER: Encrypt six columns, with different keys.
* Do not encrypt footer (to enable legacy
readers)
* - plaintext footer mode.
+ * ENCRYPT_COLUMNS_PLAINTEXT_FOOTER_COMPLETE: Encrypt six columns with
different keys
+ * Do not encrypt footer. Encrypt the rest of
the columns
+ * with the footer key.
* ENCRYPT_COLUMNS_AND_FOOTER_AAD: Encrypt six columns and the footer, with
different
* keys. Supply aad_prefix for file identity
* verification.
@@ -255,6 +254,21 @@ public class TestEncryptionOptions {
.withFooterKeyMetadata(footerKeyMetadata).build();
}
},
+ ENCRYPT_COLUMNS_PLAIN_FOOTER_COMPLETE {
+ /**
+ * Encryption configuration 8: Encrypt six columns with different keys.
+ * Encrypt the rest of the columns with the footer key. Don't encrypt
footer.
+ */
+ public FileEncryptionProperties getEncryptionProperties() {
+ Map<ColumnPath, ColumnEncryptionProperties> columnPropertiesMap =
getColumnEncryptionPropertiesMap();
+ return FileEncryptionProperties.builder(FOOTER_ENCRYPTION_KEY)
+ .withFooterKeyMetadata(footerKeyMetadata)
+ .withEncryptedColumns(columnPropertiesMap)
+ .withCompleteColumnEncryption()
+ .withPlaintextFooter()
+ .build();
+ }
+ },
NO_ENCRYPTION {
public FileEncryptionProperties getEncryptionProperties() {
return null;
@@ -402,6 +416,14 @@ public class TestEncryptionOptions {
.named("FormatTestObject").toString());
}
+ // Project column encrypted with footer key
+ if ((decryptionConfiguration == DecryptionConfiguration.NO_DECRYPTION)
&&
+ (encryptionConfiguration ==
EncryptionConfiguration.ENCRYPT_COLUMNS_PLAIN_FOOTER_COMPLETE)) {
+ conf.set("parquet.read.schema", Types.buildMessage()
+ .optional(INT32).named(SingleRow.PLAINTEXT_INT32_FIELD_NAME)
+ .named("FormatTestObject").toString());
+ }
+
int rowNum = 0;
try (ParquetReader<Group> reader = ParquetReader.builder(new
GroupReadSupport(), file)
.withConf(conf)
@@ -471,6 +493,9 @@ public class TestEncryptionOptions {
if (EncryptionConfiguration.UNIFORM_ENCRYPTION_PLAINTEXT_FOOTER ==
encryptionConfiguration) {
continue;
}
+ if (EncryptionConfiguration.ENCRYPT_COLUMNS_PLAIN_FOOTER_COMPLETE ==
encryptionConfiguration) {
+ continue;
+ }
String fileName = getFileName(encryptionConfiguration);
Path file = new Path(rootPath, fileName);
if (!fs.exists(file)) {
@@ -500,6 +525,9 @@ public class TestEncryptionOptions {
if (EncryptionConfiguration.UNIFORM_ENCRYPTION_PLAINTEXT_FOOTER ==
encryptionConfiguration) {
continue;
}
+ if (EncryptionConfiguration.ENCRYPT_COLUMNS_PLAIN_FOOTER_COMPLETE ==
encryptionConfiguration) {
+ continue;
+ }
Path file = new Path(root, getFileName(encryptionConfiguration));
LOG.info("==> Decryption configuration {}", decryptionConfiguration);
FileDecryptionProperties fileDecryptionProperties =
decryptionConfiguration.getDecryptionProperties();
@@ -593,7 +621,7 @@ public class TestEncryptionOptions {
return;
}
}
- // Encryption_configuration 7 has null encryptor, so parquet is plaintext.
+ // Last encryption_configuration has null encryptor, so parquet is
plaintext.
// An exception is expected to be thrown if the file is being decrypted.
if (encryptionConfiguration == EncryptionConfiguration.NO_ENCRYPTION) {
if ((decryptionConfiguration ==
DecryptionConfiguration.DECRYPT_WITH_KEY_RETRIEVER) ||
