[
https://issues.apache.org/jira/browse/PARQUET-1807?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17129444#comment-17129444
]
ASF GitHub Bot commented on PARQUET-1807:
-----------------------------------------
gszadovszky commented on a change in pull request #782:
URL: https://github.com/apache/parquet-mr/pull/782#discussion_r436619593
##########
File path: .gitmodules
##########
@@ -0,0 +1,3 @@
+[submodule "parquet-hadoop/submodules/parquet-testing"]
Review comment:
Thanks for trying out the move. Hope, it'll work.
The main issue with the git submodule is it does not seem to work
automatically. My concern is if this change goes in we will have a lot of
issues from developers who cannot build parquet. We need to do something that
ensures no regression will occur with our manual (and automatic) build process.
##########
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/DecryptionKeyRetrieverMock.java
##########
@@ -0,0 +1,43 @@
+package org.apache.parquet.crypto;
Review comment:
It is strange that the RAT plugin (used for checking the license
headers) did not catch this. Anyway, please add the license header at the very
beginning of the file and the package after it.
##########
File path:
parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
##########
@@ -125,150 +133,337 @@
@Rule
public ErrorCollector errorCollector = new ErrorCollector();
+ private static String PARQUET_TESTING_PATH =
"../submodules/parquet-testing/data";
+ private static final int RANDOM_SEED = 42;
+ private static final int FIXED_LENGTH = 10;
+ private static final Random RANDOM = new Random(RANDOM_SEED);
+ private static final RandomValues.IntGenerator intGenerator
+ = new RandomValues.IntGenerator(RANDOM_SEED);
+ private static final RandomValues.FloatGenerator floatGenerator
+ = new RandomValues.FloatGenerator(RANDOM_SEED);
+ private static final RandomValues.DoubleGenerator doubleGenerator
+ = new RandomValues.DoubleGenerator(RANDOM_SEED);
+ private static final RandomValues.BinaryGenerator binaryGenerator
+ = new RandomValues.BinaryGenerator(RANDOM_SEED);
+ private static final RandomValues.FixedGenerator fixedBinaryGenerator
+ = new RandomValues.FixedGenerator(RANDOM_SEED, FIXED_LENGTH);
+
private static final byte[] FOOTER_ENCRYPTION_KEY =
"0123456789012345".getBytes();
- private static final byte[] COLUMN_ENCRYPTION_KEY1 =
"1234567890123450".getBytes();
- private static final byte[] COLUMN_ENCRYPTION_KEY2 =
"1234567890123451".getBytes();
+ private static final byte[][] COLUMN_ENCRYPTION_KEYS = {
"1234567890123450".getBytes(),
+ "1234567890123451".getBytes(), "1234567890123452".getBytes(),
"1234567890123453".getBytes(),
+ "1234567890123454".getBytes(), "1234567890123455".getBytes()};
+ private static final String[] COLUMN_ENCRYPTION_KEY_IDS = { "kc1", "kc2",
"kc3", "kc4", "kc5", "kc6"};
private static final String FOOTER_ENCRYPTION_KEY_ID = "kf";
- private static final String COLUMN_ENCRYPTION_KEY1_ID = "kc1";
- private static final String COLUMN_ENCRYPTION_KEY2_ID = "kc2";
private static final String AAD_PREFIX_STRING = "tester";
private static final String BOOLEAN_FIELD_NAME = "boolean_field";
private static final String INT32_FIELD_NAME = "int32_field";
private static final String FLOAT_FIELD_NAME = "float_field";
private static final String DOUBLE_FIELD_NAME = "double_field";
+ private static final String BINARY_FIELD_NAME = "ba_field";
+ private static final String FIXED_LENGTH_BINARY_FIELD_NAME = "flba_field";
+ private static final String PLAINTEXT_INT32_FIELD_NAME = "plain_int32_field";
+
+ private static final byte[] footerKeyMetadata =
FOOTER_ENCRYPTION_KEY_ID.getBytes(StandardCharsets.UTF_8);
+ private static final byte[] AADPrefix =
AAD_PREFIX_STRING.getBytes(StandardCharsets.UTF_8);
+
+ private static final int ROW_COUNT = 10000;
+ private static final List<SingleRow> DATA =
Collections.unmodifiableList(generateRandomData(ROW_COUNT));
+ private static final List<SingleRow> LINEAR_DATA =
Collections.unmodifiableList(generateLinearData(250));
+
+ private static final MessageType SCHEMA =
+ new MessageType("schema",
+ new PrimitiveType(REQUIRED, BOOLEAN, BOOLEAN_FIELD_NAME),
+ new PrimitiveType(REQUIRED, INT32, INT32_FIELD_NAME),
+ new PrimitiveType(REQUIRED, FLOAT, FLOAT_FIELD_NAME),
+ new PrimitiveType(REQUIRED, DOUBLE, DOUBLE_FIELD_NAME),
+ new PrimitiveType(OPTIONAL, BINARY, BINARY_FIELD_NAME),
+
Types.required(FIXED_LEN_BYTE_ARRAY).length(FIXED_LENGTH).named(FIXED_LENGTH_BINARY_FIELD_NAME),
+ new PrimitiveType(OPTIONAL, INT32, PLAINTEXT_INT32_FIELD_NAME));
+
+ private static final DecryptionKeyRetrieverMock decryptionKeyRetrieverMock =
new DecryptionKeyRetrieverMock()
+ .putKey(FOOTER_ENCRYPTION_KEY_ID, FOOTER_ENCRYPTION_KEY)
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[0], COLUMN_ENCRYPTION_KEYS[0])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[1], COLUMN_ENCRYPTION_KEYS[1])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[2], COLUMN_ENCRYPTION_KEYS[2])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[3], COLUMN_ENCRYPTION_KEYS[3])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[4], COLUMN_ENCRYPTION_KEYS[4])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[5], COLUMN_ENCRYPTION_KEYS[5]);
public enum EncryptionConfiguration {
- UNIFORM_ENCRYPTION("UNIFORM_ENCRYPTION"),
- ENCRYPT_COLUMNS_AND_FOOTER("ENCRYPT_COLUMNS_AND_FOOTER"),
- ENCRYPT_COLUMNS_PLAINTEXT_FOOTER("ENCRYPT_COLUMNS_PLAINTEXT_FOOTER"),
- ENCRYPT_COLUMNS_AND_FOOTER_AAD("ENCRYPT_COLUMNS_AND_FOOTER_AAD"),
-
ENCRYPT_COLUMNS_AND_FOOTER_DISABLE_AAD_STORAGE("ENCRYPT_COLUMNS_AND_FOOTER_DISABLE_AAD_STORAGE"),
- ENCRYPT_COLUMNS_AND_FOOTER_CTR("ENCRYPT_COLUMNS_AND_FOOTER_CTR"),
- NO_ENCRYPTION("NO_ENCRYPTION");
-
- private final String configurationName;
-
- EncryptionConfiguration(String configurationName) {
- this.configurationName = configurationName;
- }
+ UNIFORM_ENCRYPTION {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getUniformEncryptionEncryptionProperties();
+ }
+ },
+ ENCRYPT_COLUMNS_AND_FOOTER {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getEncryptColumnsAndFooterEncryptionProperties();
+ }
+ },
+ ENCRYPT_COLUMNS_PLAINTEXT_FOOTER {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getPlaintextFooterEncryptionProperties();
+ }
+ },
+ ENCRYPT_COLUMNS_AND_FOOTER_AAD {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getEncryptWithAADEncryptionProperties();
+ }
+ },
+ ENCRYPT_COLUMNS_AND_FOOTER_DISABLE_AAD_STORAGE {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getDisableAADStorageEncryptionProperties();
+ }
+ },
+ ENCRYPT_COLUMNS_AND_FOOTER_CTR {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getCTREncryptionProperties();
+ }
+ },
+ NO_ENCRYPTION {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return null;
+ }
+ };
- @Override
- public String toString() {
- return configurationName;
- }
+ abstract public FileEncryptionProperties getEncryptionProperties();
}
-
public enum DecryptionConfiguration {
- DECRYPT_WITH_KEY_RETRIEVER("DECRYPT_WITH_KEY_RETRIEVER"),
- DECRYPT_WITH_KEY_RETRIEVER_AAD("DECRYPT_WITH_KEY_RETRIEVER_AAD"),
- DECRYPT_WITH_EXPLICIT_KEYS("DECRYPT_WITH_EXPLICIT_KEYS"),
- NO_DECRYPTION("NO_DECRYPTION");
-
- private final String configurationName;
-
- DecryptionConfiguration(String configurationName) {
- this.configurationName = configurationName;
- }
+ DECRYPT_WITH_KEY_RETRIEVER {
+ public FileDecryptionProperties getDecryptionProperties() {
+ return getKeyRetrieverDecryptionProperties();
+ }
+ },
+ DECRYPT_WITH_KEY_RETRIEVER_AAD {
+ public FileDecryptionProperties getDecryptionProperties() {
+ return getKeyRetrieverAADDecryptionProperties();
+ }
+ },
+ DECRYPT_WITH_EXPLICIT_KEYS {
+ public FileDecryptionProperties getDecryptionProperties() {
+ return getExplicitKeysDecryptionProperties();
+ }
+ },
+ NO_DECRYPTION {
+ public FileDecryptionProperties getDecryptionProperties() {
+ return null;
+ }
+ };
- @Override
- public String toString() {
- return configurationName;
- }
+ abstract public FileDecryptionProperties getDecryptionProperties();
}
@Test
public void testWriteReadEncryptedParquetFiles() throws IOException {
Path rootPath = new Path(temporaryFolder.getRoot().getPath());
LOG.info("======== testWriteReadEncryptedParquetFiles {} ========",
rootPath.toString());
byte[] AADPrefix = AAD_PREFIX_STRING.getBytes(StandardCharsets.UTF_8);
- // This array will hold various encryption configuraions.
- Map<EncryptionConfiguration, FileEncryptionProperties>
encryptionPropertiesMap =
- getEncryptionConfigurations(AADPrefix);
- testWriteEncryptedParquetFiles(rootPath, encryptionPropertiesMap);
- // This array will hold various decryption configurations.
- Map<DecryptionConfiguration, FileDecryptionProperties>
decryptionPropertiesMap =
- getDecryptionConfigurations(AADPrefix);
- testReadEncryptedParquetFiles(rootPath, decryptionPropertiesMap);
+ // Write using various encryption configuraions
+ testWriteEncryptedParquetFiles(rootPath, DATA);
+ // Read using various decryption configurations.
+ testReadEncryptedParquetFiles(rootPath, DATA);
}
@Test
public void testInteropReadEncryptedParquetFiles() throws IOException {
- Path rootPath = new Path("submodules/parquet-testing/data");
+ Path rootPath = new Path(PARQUET_TESTING_PATH);
LOG.info("======== testInteropReadEncryptedParquetFiles {} ========",
rootPath.toString());
byte[] AADPrefix = AAD_PREFIX_STRING.getBytes(StandardCharsets.UTF_8);
- // This array will hold various decryption configurations.
- Map<DecryptionConfiguration, FileDecryptionProperties>
decryptionPropertiesMap =
- getDecryptionConfigurations(AADPrefix);
- testReadEncryptedParquetFiles(rootPath, decryptionPropertiesMap);
+ // Read using various decryption configurations.
+ testInteropReadEncryptedParquetFiles(rootPath, true/*readOnlyEncrypted*/,
LINEAR_DATA);
}
- private void testWriteEncryptedParquetFiles(Path root,
Map<EncryptionConfiguration, FileEncryptionProperties> encryptionPropertiesMap)
throws IOException {
- Configuration conf = new Configuration();
- int numberOfEncryptionModes = encryptionPropertiesMap.size();
+ private static List<SingleRow> generateRandomData(int rowCount) {
+ List<SingleRow> dataList = new ArrayList<>(rowCount);
+ for (int row = 0; row < rowCount; ++row) {
+ SingleRow newRow = new SingleRow(RANDOM.nextBoolean(),
+ intGenerator.nextValue(), floatGenerator.nextValue(),
+ doubleGenerator.nextValue(), binaryGenerator.nextValue().getBytes(),
+ fixedBinaryGenerator.nextValue().getBytes(), intGenerator.nextValue());
+ dataList.add(newRow);
+ }
+ return dataList;
+ }
+
+ private static List<SingleRow> generateLinearData(int rowCount) {
+ List<SingleRow> dataList = new ArrayList<>(rowCount);
+ String baseStr = "parquet";
+ for (int row = 0; row < rowCount; ++row) {
+ boolean boolean_val = ((row % 2) == 0) ? true : false;
+ float float_val = (float) row * 1.1f;
+ double double_val = (row * 1.1111111);
+
+ byte[] binary_val = null;
+ if ((row % 2) == 0) {
+ char firstChar = (char) ((int) '0' + row / 100);
+ char secondChar = (char) ((int) '0' + (row / 10) % 10);
+ char thirdChar = (char) ((int) '0' + row % 10);
+ binary_val = (baseStr + firstChar + secondChar +
thirdChar).getBytes(StandardCharsets.UTF_8);
+ }
+ char[] fixed = new char[FIXED_LENGTH];
+ char[] aChar = Character.toChars(row);
+ Arrays.fill(fixed, aChar[0]);
+
+ SingleRow newRow = new SingleRow(boolean_val,
+ row, float_val, double_val,
+ binary_val, new String(fixed).getBytes(StandardCharsets.UTF_8),
null/*plaintext_int32_field*/);
+ dataList.add(newRow);
+ }
+ return dataList;
+ }
- MessageType schema = parseMessageType(
- "message test { "
- + "required boolean " + BOOLEAN_FIELD_NAME + "; "
- + "required int32 " + INT32_FIELD_NAME + "; "
- + "required float " + FLOAT_FIELD_NAME + "; "
- + "required double " + DOUBLE_FIELD_NAME + "; "
- + "} ");
+ public static class SingleRow {
+ public final boolean boolean_field;
+ public final int int32_field;
+ public final float float_field;
+ public final double double_field;
+ public final byte[] ba_field;
+ public final byte[] flba_field;
+ public final Integer plaintext_int32_field; // Can be null, since it
doesn't exist in C++-created files yet.
+
+ public SingleRow(boolean boolean_field,
+ int int32_field,
+ float float_field,
+ double double_field,
+ byte[] ba_field,
+ byte[] flba_field,
+ Integer plaintext_int32_field) {
+ this.boolean_field = boolean_field;
+ this.int32_field = int32_field;
+ this.float_field = float_field;
+ this.double_field = double_field;
+ this.ba_field = ba_field;
+ this.flba_field = flba_field;
+ this.plaintext_int32_field = plaintext_int32_field;
+ }
+ }
- GroupWriteSupport.setSchema(schema, conf);
- SimpleGroupFactory f = new SimpleGroupFactory(schema);
+ private void testWriteEncryptedParquetFiles(Path root, List<SingleRow> data)
throws IOException {
+ Configuration conf = new Configuration();
+ int pageSize = data.size() / 10; // Ensure that several pages will be
created
+ int rowGroupSize = pageSize * 6 * 5; // Ensure that there are more
row-groups created
- for (Map.Entry<EncryptionConfiguration, FileEncryptionProperties>
encryptionConfigurationEntry : encryptionPropertiesMap.entrySet()) {
- EncryptionConfiguration encryptionConfiguration =
encryptionConfigurationEntry.getKey();
- Path file = new Path(root, encryptionConfiguration.toString() +
".parquet.encrypted");
+ SimpleGroupFactory f = new SimpleGroupFactory(SCHEMA);
+ EncryptionConfiguration[] encryptionConfigurations =
EncryptionConfiguration.values();
+ for (EncryptionConfiguration encryptionConfiguration :
encryptionConfigurations) {
+ Path file = new Path(root, getFileName(encryptionConfiguration));
+ FileEncryptionProperties encryptionProperties =
encryptionConfiguration.getEncryptionProperties();
LOG.info("\nWrite " + file.toString());
- ParquetWriter<Group> writer = ExampleParquetWriter.builder(file)
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(file)
.withWriteMode(OVERWRITE)
- .withType(schema)
- .withEncryption(encryptionConfigurationEntry.getValue())
- .build();
-
- for (int i = 0; i < 100; i++) {
- boolean expect = false;
- if ((i % 2) == 0)
- expect = true;
- float float_val = (float) i * 1.1f;
- double double_val = (i * 1.1111111);
-
- writer.write(
- f.newGroup()
- .append(BOOLEAN_FIELD_NAME, expect)
- .append(INT32_FIELD_NAME, i)
- .append(FLOAT_FIELD_NAME, float_val)
- .append(DOUBLE_FIELD_NAME, double_val));
+ .withRowGroupSize(rowGroupSize)
+ .withPageSize(pageSize)
+ .withType(SCHEMA)
+ .withConf(conf)
+ .withEncryption(encryptionProperties)
+ .build()) {
+
+ for (SingleRow singleRow : data) {
+ writer.write(
+ f.newGroup()
+ .append(BOOLEAN_FIELD_NAME, singleRow.boolean_field)
+ .append(INT32_FIELD_NAME, singleRow.int32_field)
+ .append(FLOAT_FIELD_NAME, singleRow.float_field)
+ .append(DOUBLE_FIELD_NAME, singleRow.double_field)
+ .append(BINARY_FIELD_NAME,
Binary.fromConstantByteArray(singleRow.ba_field))
+ .append(FIXED_LENGTH_BINARY_FIELD_NAME,
Binary.fromConstantByteArray(singleRow.flba_field))
+ .append(PLAINTEXT_INT32_FIELD_NAME,
singleRow.plaintext_int32_field));
+ }
}
- writer.close();
}
}
- private void testReadEncryptedParquetFiles(Path root,
Map<DecryptionConfiguration, FileDecryptionProperties> decryptionPropertiesMap)
throws IOException {
+ private String getFileName(EncryptionConfiguration encryptionConfiguration) {
+ return encryptionConfiguration.toString().toLowerCase() +
".parquet.encrypted";
+ }
+
+ private void testReadEncryptedParquetFiles(Path root, List<SingleRow> data) {
Configuration conf = new Configuration();
+ DecryptionConfiguration[] decryptionConfigurations =
DecryptionConfiguration.values();
+ for (DecryptionConfiguration decryptionConfiguration :
decryptionConfigurations) {
+ EncryptionConfiguration[] encryptionConfigurations =
EncryptionConfiguration.values();
+ for (EncryptionConfiguration encryptionConfiguration :
encryptionConfigurations) {
+ Path file = new Path(root, getFileName(encryptionConfiguration));
+ LOG.info("==> Decryption configuration {}", decryptionConfiguration);
+ FileDecryptionProperties fileDecryptionProperties =
decryptionConfiguration.getDecryptionProperties();
- for (Map.Entry<DecryptionConfiguration, FileDecryptionProperties>
decryptionConfigurationEntry : decryptionPropertiesMap.entrySet()) {
- DecryptionConfiguration decryptionConfiguration =
decryptionConfigurationEntry.getKey();
- LOG.info("==> Decryption configuration {}", decryptionConfiguration);
- FileDecryptionProperties fileDecryptionProperties =
decryptionConfigurationEntry.getValue();
+ LOG.info("--> Read file {} {}", file.toString(),
encryptionConfiguration);
- File folder = new File(root.toString());
- File[] listOfFiles = folder.listFiles();
+ // Read only the non-encrypted columns
+ if ((decryptionConfiguration == DecryptionConfiguration.NO_DECRYPTION)
&&
+ (encryptionConfiguration ==
EncryptionConfiguration.ENCRYPT_COLUMNS_PLAINTEXT_FOOTER)) {
+ conf.set("parquet.read.schema", Types.buildMessage()
+ .optional(INT32).named(PLAINTEXT_INT32_FIELD_NAME)
+ .named("FormatTestObject").toString());
+ }
- for (int fileNum = 0; fileNum < listOfFiles.length; fileNum++) {
- Path file = new Path(listOfFiles[fileNum].getAbsolutePath());
- if (!file.getName().endsWith("parquet.encrypted")) { // Skip non
encrypted files
- continue;
+ int rowNum = 0;
+ try (ParquetReader<Group> reader = ParquetReader.builder(new
GroupReadSupport(), file)
+ .withConf(conf)
+ .withDecryption(fileDecryptionProperties)
+ .build()) {
+ for (Group group = reader.read(); group != null; group =
reader.read()) {
+ SingleRow rowExpected = data.get(rowNum++);
+ // plaintext columns
+ if ((null != rowExpected.plaintext_int32_field) &&
+ rowExpected.plaintext_int32_field !=
group.getInteger(PLAINTEXT_INT32_FIELD_NAME, 0)) {
+ addErrorToErrorCollectorAndLog("Wrong int",
encryptionConfiguration, decryptionConfiguration);
+ }
Review comment:
What do you think about this way?
```
import static org.hamcrest.CoreMatchers.equalTo;
// ...
if (null != rowExpected.plaintext_int32_field)
errorCollector.checkThat("Wrong int",
group.getInteger(PLAINTEXT_INT32_FIELD_NAME, 0),
equalTo(rowExpected.plaintext_int32_field));
}
```
Similarly, you may do the check like that in any other places just like you
would do with `assertEquals`. (You may find a couple of other _matchers_ as
well.)
UPDATE: Meanwhile, I realized that you are not logging the values but the
crypto configurations which are more relevant for this test. So, you may either
skip my comment above or use it for creating another
`addErrorToCollectorAndLog` like method.
BTW, why do we need to check the expected value for `null`? Shouldn't be the
actual value `null` as well?
##########
File path:
parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
##########
@@ -125,150 +133,337 @@
@Rule
public ErrorCollector errorCollector = new ErrorCollector();
+ private static String PARQUET_TESTING_PATH =
"../submodules/parquet-testing/data";
+ private static final int RANDOM_SEED = 42;
+ private static final int FIXED_LENGTH = 10;
+ private static final Random RANDOM = new Random(RANDOM_SEED);
+ private static final RandomValues.IntGenerator intGenerator
+ = new RandomValues.IntGenerator(RANDOM_SEED);
+ private static final RandomValues.FloatGenerator floatGenerator
+ = new RandomValues.FloatGenerator(RANDOM_SEED);
+ private static final RandomValues.DoubleGenerator doubleGenerator
+ = new RandomValues.DoubleGenerator(RANDOM_SEED);
+ private static final RandomValues.BinaryGenerator binaryGenerator
+ = new RandomValues.BinaryGenerator(RANDOM_SEED);
+ private static final RandomValues.FixedGenerator fixedBinaryGenerator
+ = new RandomValues.FixedGenerator(RANDOM_SEED, FIXED_LENGTH);
+
private static final byte[] FOOTER_ENCRYPTION_KEY =
"0123456789012345".getBytes();
- private static final byte[] COLUMN_ENCRYPTION_KEY1 =
"1234567890123450".getBytes();
- private static final byte[] COLUMN_ENCRYPTION_KEY2 =
"1234567890123451".getBytes();
+ private static final byte[][] COLUMN_ENCRYPTION_KEYS = {
"1234567890123450".getBytes(),
+ "1234567890123451".getBytes(), "1234567890123452".getBytes(),
"1234567890123453".getBytes(),
+ "1234567890123454".getBytes(), "1234567890123455".getBytes()};
+ private static final String[] COLUMN_ENCRYPTION_KEY_IDS = { "kc1", "kc2",
"kc3", "kc4", "kc5", "kc6"};
private static final String FOOTER_ENCRYPTION_KEY_ID = "kf";
- private static final String COLUMN_ENCRYPTION_KEY1_ID = "kc1";
- private static final String COLUMN_ENCRYPTION_KEY2_ID = "kc2";
private static final String AAD_PREFIX_STRING = "tester";
private static final String BOOLEAN_FIELD_NAME = "boolean_field";
private static final String INT32_FIELD_NAME = "int32_field";
private static final String FLOAT_FIELD_NAME = "float_field";
private static final String DOUBLE_FIELD_NAME = "double_field";
+ private static final String BINARY_FIELD_NAME = "ba_field";
+ private static final String FIXED_LENGTH_BINARY_FIELD_NAME = "flba_field";
+ private static final String PLAINTEXT_INT32_FIELD_NAME = "plain_int32_field";
+
+ private static final byte[] footerKeyMetadata =
FOOTER_ENCRYPTION_KEY_ID.getBytes(StandardCharsets.UTF_8);
+ private static final byte[] AADPrefix =
AAD_PREFIX_STRING.getBytes(StandardCharsets.UTF_8);
+
+ private static final int ROW_COUNT = 10000;
+ private static final List<SingleRow> DATA =
Collections.unmodifiableList(generateRandomData(ROW_COUNT));
+ private static final List<SingleRow> LINEAR_DATA =
Collections.unmodifiableList(generateLinearData(250));
+
+ private static final MessageType SCHEMA =
+ new MessageType("schema",
+ new PrimitiveType(REQUIRED, BOOLEAN, BOOLEAN_FIELD_NAME),
+ new PrimitiveType(REQUIRED, INT32, INT32_FIELD_NAME),
+ new PrimitiveType(REQUIRED, FLOAT, FLOAT_FIELD_NAME),
+ new PrimitiveType(REQUIRED, DOUBLE, DOUBLE_FIELD_NAME),
+ new PrimitiveType(OPTIONAL, BINARY, BINARY_FIELD_NAME),
+
Types.required(FIXED_LEN_BYTE_ARRAY).length(FIXED_LENGTH).named(FIXED_LENGTH_BINARY_FIELD_NAME),
+ new PrimitiveType(OPTIONAL, INT32, PLAINTEXT_INT32_FIELD_NAME));
+
+ private static final DecryptionKeyRetrieverMock decryptionKeyRetrieverMock =
new DecryptionKeyRetrieverMock()
+ .putKey(FOOTER_ENCRYPTION_KEY_ID, FOOTER_ENCRYPTION_KEY)
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[0], COLUMN_ENCRYPTION_KEYS[0])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[1], COLUMN_ENCRYPTION_KEYS[1])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[2], COLUMN_ENCRYPTION_KEYS[2])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[3], COLUMN_ENCRYPTION_KEYS[3])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[4], COLUMN_ENCRYPTION_KEYS[4])
+ .putKey(COLUMN_ENCRYPTION_KEY_IDS[5], COLUMN_ENCRYPTION_KEYS[5]);
public enum EncryptionConfiguration {
- UNIFORM_ENCRYPTION("UNIFORM_ENCRYPTION"),
- ENCRYPT_COLUMNS_AND_FOOTER("ENCRYPT_COLUMNS_AND_FOOTER"),
- ENCRYPT_COLUMNS_PLAINTEXT_FOOTER("ENCRYPT_COLUMNS_PLAINTEXT_FOOTER"),
- ENCRYPT_COLUMNS_AND_FOOTER_AAD("ENCRYPT_COLUMNS_AND_FOOTER_AAD"),
-
ENCRYPT_COLUMNS_AND_FOOTER_DISABLE_AAD_STORAGE("ENCRYPT_COLUMNS_AND_FOOTER_DISABLE_AAD_STORAGE"),
- ENCRYPT_COLUMNS_AND_FOOTER_CTR("ENCRYPT_COLUMNS_AND_FOOTER_CTR"),
- NO_ENCRYPTION("NO_ENCRYPTION");
-
- private final String configurationName;
-
- EncryptionConfiguration(String configurationName) {
- this.configurationName = configurationName;
- }
+ UNIFORM_ENCRYPTION {
+ public FileEncryptionProperties getEncryptionProperties() {
+ return getUniformEncryptionEncryptionProperties();
Review comment:
It is more a coding style question but I think it would be more readable
to simple have the implementation here in the enums instead of adding new
static methods. If you like it more as is I'm also fine with it.
##########
File path: dev/submodule-update.sh
##########
@@ -0,0 +1,25 @@
+#!/bin/bash
Review comment:
Unfortunately, there are developers who build parquet in a Windows
environment. At least, I remember some issues related to Windows. I would
suggest executing `git submodule update` directly from the `exec-maven-plugin`
is it will be platform independent. (See
[here](https://stackoverflow.com/a/29796988/13573181) for reference.)
##########
File path:
parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java
##########
@@ -657,4 +617,201 @@ private void addErrorToErrorCollectorAndLog(String
errorMessage, EncryptionConfi
errorCollector.addError(new Throwable(fullErrorMessage));
LOG.error(fullErrorMessage);
}
+
+ private static Map<ColumnPath, ColumnEncryptionProperties>
getColumnEncryptionPropertiesMap() {
+ Map<ColumnPath, ColumnEncryptionProperties> columnPropertiesMap = new
HashMap<>();
+
+ ColumnEncryptionProperties columnPropertiesDouble =
ColumnEncryptionProperties
+ .builder(DOUBLE_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[0])
+ .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[0])
+ .build();
+ columnPropertiesMap.put(columnPropertiesDouble.getPath(),
columnPropertiesDouble);
+
+ ColumnEncryptionProperties columnPropertiesFloat =
ColumnEncryptionProperties
+ .builder(FLOAT_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[1])
+ .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[1])
+ .build();
+ columnPropertiesMap.put(columnPropertiesFloat.getPath(),
columnPropertiesFloat);
+
+ ColumnEncryptionProperties columnPropertiesBool =
ColumnEncryptionProperties
+ .builder(BOOLEAN_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[2])
+ .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[2])
+ .build();
+ columnPropertiesMap.put(columnPropertiesBool.getPath(),
columnPropertiesBool);
+
+ ColumnEncryptionProperties columnPropertiesInt32 =
ColumnEncryptionProperties
+ .builder(INT32_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[3])
+ .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[3])
+ .build();
+ columnPropertiesMap.put(columnPropertiesInt32.getPath(),
columnPropertiesInt32);
+
+ ColumnEncryptionProperties columnPropertiesBinary =
ColumnEncryptionProperties
+ .builder(BINARY_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[4])
+ .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[4])
+ .build();
+ columnPropertiesMap.put(columnPropertiesBinary.getPath(),
columnPropertiesBinary);
+
+ ColumnEncryptionProperties columnPropertiesFixed =
ColumnEncryptionProperties
+ .builder(FIXED_LENGTH_BINARY_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[5])
+ .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[5])
+ .build();
+ columnPropertiesMap.put(columnPropertiesFixed.getPath(),
columnPropertiesFixed);
+
+ return columnPropertiesMap;
+ }
+
+ private static Map<ColumnPath, ColumnDecryptionProperties>
getColumnDecryptionPropertiesMap() {
+ Map<ColumnPath, ColumnDecryptionProperties> columnMap = new HashMap<>();
+
+ ColumnDecryptionProperties columnDecryptionPropsDouble =
ColumnDecryptionProperties
+ .builder(DOUBLE_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[0])
+ .build();
+ columnMap.put(columnDecryptionPropsDouble.getPath(),
columnDecryptionPropsDouble);
+
+ ColumnDecryptionProperties columnDecryptionPropsFloat =
ColumnDecryptionProperties
+ .builder(FLOAT_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[1])
+ .build();
+ columnMap.put(columnDecryptionPropsFloat.getPath(),
columnDecryptionPropsFloat);
+
+ ColumnDecryptionProperties columnDecryptionPropsBool =
ColumnDecryptionProperties
+ .builder(BOOLEAN_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[2])
+ .build();
+ columnMap.put(columnDecryptionPropsBool.getPath(),
columnDecryptionPropsBool);
+
+ ColumnDecryptionProperties columnDecryptionPropsInt32 =
ColumnDecryptionProperties
+ .builder(INT32_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[3])
+ .build();
+ columnMap.put(columnDecryptionPropsInt32.getPath(),
columnDecryptionPropsInt32);
+
+ ColumnDecryptionProperties columnDecryptionPropsBinary =
ColumnDecryptionProperties
+ .builder(BINARY_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[4])
+ .build();
+ columnMap.put(columnDecryptionPropsBinary.getPath(),
columnDecryptionPropsBinary);
+
+ ColumnDecryptionProperties columnDecryptionPropsFixed =
ColumnDecryptionProperties
+ .builder(FIXED_LENGTH_BINARY_FIELD_NAME)
+ .withKey(COLUMN_ENCRYPTION_KEYS[5])
+ .build();
+ columnMap.put(columnDecryptionPropsFixed.getPath(),
columnDecryptionPropsFixed);
+
+ return columnMap;
+ }
+
+ /**
+ * Encryption configuration 1: Encrypt all columns and the footer with the
same key.
+ */
+ private static FileEncryptionProperties
getUniformEncryptionEncryptionProperties() {
Review comment:
nit:
```suggestion
private static FileEncryptionProperties getUniformEncryptionProperties() {
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Encryption: Interop and Function test suite for Java version
> ------------------------------------------------------------
>
> Key: PARQUET-1807
> URL: https://issues.apache.org/jira/browse/PARQUET-1807
> Project: Parquet
> Issue Type: Sub-task
> Components: parquet-mr
> Reporter: Gidon Gershinsky
> Assignee: Maya Anderson
> Priority: Major
> Labels: pull-request-available
>
> # Interop: test parquet-mr reading of encrypted files produced by parquet-cpp
> (fetched from parquet-testing)
> # Function: test writing/reading in a number of encryption and decryption
> configurations
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
