[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r510460619
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
##
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto.propertiesfactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.example.GroupWriteSupport;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+import static org.apache.parquet.schema.Type.Repetition.OPTIONAL;
+import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class SchemaControlEncryptionTest {
+
+ private final static Log LOG =
LogFactory.getLog(SchemaControlEncryptionTest.class);
+ private final static int numRecord = 1000;
+ private Random rnd = new Random(5);
+
+ // In the test We use a map to tell WriteSupport which columns to be
encrypted with what key. In real use cases, people
+ // can find whatever easy way to do so basing on how do they get these
information, for example people can choose to
+ // store in HMS, or other metastore.
+ private Map> crytoMetadata = new HashMap<>();
+ private Map testData = new HashMap<>();
+
+ @Before
+ public void generateTestData() {
+String[] names = new String[numRecord];
+Long[] ages = new Long[numRecord];
+String[] linkedInWebs = new String[numRecord];
+String[] twitterWebs = new String[numRecord];
+for (int i = 0; i < numRecord; i++) {
+ names[i] = getString();
+ ages[i] = getLong();
+ linkedInWebs[i] = getString();
+ twitterWebs[i] = getString();
+}
+
+testData.put("Name", names);
+testData.put("Age", ages);
+testData.put("LinkedIn", linkedInWebs);
+testData.put("Twitter", twitterWebs);
+ }
+
+ @Test
+ public void testEncryptionDefault() throws Exception {
+Configuration conf = new Configuration();
+runTest(conf);
+ }
+
+ @Test
+ public void testEncryptionGcm() throws Exception {
+Configuration conf = new Configuration();
+conf.set(SchemaCryptoPropertiesFactory.CONF_ENCRYPTION_ALGORITHM,
"AES_GCM_CTR_V1");
+runTest(conf);
+ }
+
+ @Test
+ public void testEncryptionGcmCtr() throws Exception {
+Configuration conf = new Configuration();
+conf.set(SchemaCryptoPropertiesFactory.CONF_ENCRYPTION_ALGORITHM,
"AES_GCM_V1");
+runTest(conf);
+ }
+
+ @Test
+ public void testEncryptionWithFooter() throws Exception {
+Configuration conf = new Configuration();
+conf.setBoolean(SchemaCryptoPropertiesFactory.CONF_ENCRYPTION_FOOTER,
true);
+runTest(conf);
+ }
+
+ private void runTest(Configuration conf ) throws Exception {
+conf.set(EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME,
+
"org.apache.parquet.crypto.propertiesfactory.SchemaCryptoPropertiesFactory");
Review comment:
changed to
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r510457820
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
##
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto.propertiesfactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.example.GroupWriteSupport;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+import static org.apache.parquet.schema.Type.Repetition.OPTIONAL;
+import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class SchemaControlEncryptionTest {
+
+ private final static Log LOG =
LogFactory.getLog(SchemaControlEncryptionTest.class);
+ private final static int numRecord = 1000;
+ private Random rnd = new Random(5);
+
+ // In the test We use a map to tell WriteSupport which columns to be
encrypted with what key. In real use cases, people
+ // can find whatever easy way to do so basing on how do they get these
information, for example people can choose to
+ // store in HMS, or other metastore.
+ private Map> crytoMetadata = new HashMap<>();
+ private Map testData = new HashMap<>();
+
+ @Before
+ public void generateTestData() {
+String[] names = new String[numRecord];
+Long[] ages = new Long[numRecord];
+String[] linkedInWebs = new String[numRecord];
+String[] twitterWebs = new String[numRecord];
+for (int i = 0; i < numRecord; i++) {
+ names[i] = getString();
+ ages[i] = getLong();
+ linkedInWebs[i] = getString();
+ twitterWebs[i] = getString();
+}
+
+testData.put("Name", names);
+testData.put("Age", ages);
+testData.put("LinkedIn", linkedInWebs);
+testData.put("Twitter", twitterWebs);
+ }
+
+ @Test
+ public void testEncryptionDefault() throws Exception {
+Configuration conf = new Configuration();
+runTest(conf);
+ }
+
+ @Test
+ public void testEncryptionGcm() throws Exception {
+Configuration conf = new Configuration();
+conf.set(SchemaCryptoPropertiesFactory.CONF_ENCRYPTION_ALGORITHM,
"AES_GCM_CTR_V1");
+runTest(conf);
+ }
+
+ @Test
+ public void testEncryptionGcmCtr() throws Exception {
+Configuration conf = new Configuration();
+conf.set(SchemaCryptoPropertiesFactory.CONF_ENCRYPTION_ALGORITHM,
"AES_GCM_V1");
Review comment:
fixed
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
##
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r510278202
##
File path:
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
##
@@ -279,6 +279,11 @@ public ParquetWriter(Path file, Configuration conf,
WriteSupport writeSupport
WriteSupport.WriteContext writeContext = writeSupport.init(conf);
MessageType schema = writeContext.getSchema();
+// encryptionProperties could be built from the implementation of
EncryptionPropertiesFactory when it is attached.
+if (encryptionProperties == null) {
+ encryptionProperties =
ParquetOutputFormat.createEncryptionProperties(conf, new Path(file.toString()),
writeContext);
Review comment:
Good point. I like the idea of adding "Path getPath()" better. Since
this feature is going to be in a major release. I think
adding a new method to the interface could be fine.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r509863658
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaCryptoPropertiesFactory.java
##
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto.propertiesfactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.crypto.ColumnEncryptionProperties;
+import org.apache.parquet.crypto.DecryptionKeyRetrieverMock;
+import org.apache.parquet.crypto.DecryptionPropertiesFactory;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.crypto.FileDecryptionProperties;
+import org.apache.parquet.crypto.FileEncryptionProperties;
+import org.apache.parquet.crypto.ParquetCipher;
+import org.apache.parquet.crypto.ParquetCryptoRuntimeException;
+import org.apache.parquet.hadoop.api.WriteSupport.WriteContext;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.schema.MessageType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SchemaCryptoPropertiesFactory implements
EncryptionPropertiesFactory, DecryptionPropertiesFactory {
+
+ private static Logger log =
LoggerFactory.getLogger(SchemaCryptoPropertiesFactory.class);
+
+ public static final String CONF_ENCRYPTION_ALGORITHM =
"parquet.encryption.algorithm";
+ public static final String CONF_ENCRYPTION_FOOTER = "parquet.encrypt.footer";
+ private static final byte[] FOOTER_KEY = {0x01, 0x02, 0x03, 0x4, 0x05, 0x06,
0x07, 0x08, 0x09, 0x0a,
+0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10};
+ private static final byte[] FOOTER_KEY_METADATA =
"footkey".getBytes(StandardCharsets.UTF_8);
+ private static final byte[] COL_KEY = {0x02, 0x03, 0x4, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b,
+0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11};
+ private static final byte[] COL_KEY_METADATA =
"col".getBytes(StandardCharsets.UTF_8);
+
+ @Override
+ public FileEncryptionProperties getFileEncryptionProperties(Configuration
conf, Path tempFilePath,
+ WriteContext
fileWriteContext) throws ParquetCryptoRuntimeException {
+MessageType schema = fileWriteContext.getSchema();
+List paths = schema.getPaths();
+if (paths == null || paths.isEmpty()) {
+ throw new ParquetCryptoRuntimeException("Null or empty fields is found");
+}
+
+Map columnPropertyMap = new
HashMap<>();
+
+for (String[] path : paths) {
+ getColumnEncryptionProperties(path, columnPropertyMap, conf);
+}
+
+if (columnPropertyMap.size() == 0) {
+ log.debug("No column is encrypted. Returning null so that Parquet can
skip. Empty properties will cause Parquet exception");
+ return null;
+}
+
+/**
+ * Why we still need footerKeyMetadata even withEncryptedFooter as false?
According to the
+ * 'Plaintext Footer' section of
+ * https://github.com/apache/parquet-format/blob/encryption/Encryption.md,
the plaintext footer
+ * is signed in order to prevent tampering with the FileMetaData contents.
So footerKeyMetadata
+ * is always needed. This signature will be verified if parquet-mr code is
with parquet-1178.
+ * Otherwise, it will be ignored.
+ */
+boolean shouldEncryptFooter = getEncryptFooter(conf);
+FileEncryptionProperties.Builder encryptionPropertiesBuilder =
+ FileEncryptionProperties.builder(FOOTER_KEY)
+.withFooterKeyMetadata(FOOTER_KEY_METADATA)
+.withAlgorithm(getParquetCipherOrDefault(conf))
+.withEncryptedColumns(columnPropertyMap);
+if (!shouldEncryptFooter) {
+ encryptionPropertiesBuilder =
encryptionPropertiesBuilder.withPlaintextFooter();
+}
+FileEncryptionProperties encryptionProperties =
encryptionPropertiesBuilder.build();
+log.info(
+ "FileEncryptionProperties is built with, algorithm:{},
footerEncrypted:{}",
+ encryptionProperties.getAlgorithm(),
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r509848961
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
##
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto.propertiesfactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.example.GroupWriteSupport;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+import static org.apache.parquet.schema.Type.Repetition.OPTIONAL;
+import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class SchemaControlEncryptionTest {
+
+ private final static Log LOG =
LogFactory.getLog(SchemaControlEncryptionTest.class);
+ private final static int numRecord = 1000;
+ private Random rnd = new Random(5);
+
+ // In the test We use a map to tell WriteSupport which columns to be
encrypted with what key. In real use cases, people
+ // can find whatever easy way to do so basing on how do they get these
information, for example people can choose to
+ // store in HMS, or other metastore.
+ private Map> crytoMetadata = new HashMap<>();
Review comment:
good catch
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r509848811
##
File path: parquet-column/src/main/java/org/apache/parquet/schema/Type.java
##
@@ -362,5 +362,4 @@ void checkContains(Type subType) {
* @return the converted tree
*/
abstract T convert(List path, TypeConverter converter);
-
-}
+ }
Review comment:
true
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r495478654
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/propertiesfactory/SchemaControlEncryptionTest.java
##
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto.propertiesfactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.example.GroupWriteSupport;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+import static org.apache.parquet.schema.Type.Repetition.OPTIONAL;
+import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class SchemaControlEncryptionTest {
+
+ private final static Log LOG =
LogFactory.getLog(SchemaControlEncryptionTest.class);
+ private final static int numRecord = 1000;
+ private Random rnd = new Random(5);
+
+ // In the test We use a map to tell WriteSupport which columns to be
encrypted with what key. In real use cases, people
+ // can find whatever easy way to do so basing on how do they get these
information, for example people can choose to
+ // store in HMS, or other metastore.
+ private Map> crytoMetadatas = new HashMap<>();
Review comment:
resolved
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r463774342
##
File path:
parquet-hadoop/src/test/java/org/apache/parquet/crypto/CryptoPropertiesFactoryTests/SchemaCryptoPropertiesFactory.java
##
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto.CryptoPropertiesFactoryTests;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.crypto.ColumnEncryptionProperties;
+import org.apache.parquet.crypto.DecryptionKeyRetrieverMock;
+import org.apache.parquet.crypto.DecryptionPropertiesFactory;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.crypto.FileDecryptionProperties;
+import org.apache.parquet.crypto.FileEncryptionProperties;
+import org.apache.parquet.crypto.ParquetCipher;
+import org.apache.parquet.crypto.ParquetCryptoRuntimeException;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.api.WriteSupport.WriteContext;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.schema.ExtType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.Type;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SchemaCryptoPropertiesFactory implements
EncryptionPropertiesFactory, DecryptionPropertiesFactory {
+
+ private static Logger log =
LoggerFactory.getLogger(SchemaCryptoPropertiesFactory.class);
+
+ public static final String CONF_ENCRYPTION_ALGORITHM =
"parquet.encryption.algorithm";
+ public static final String CONF_ENCRYPTION_FOOTER = "parquet.encrypt.footer";
+ private static final byte[] FOOTER_KEY = {0x01, 0x02, 0x03, 0x4, 0x05, 0x06,
0x07, 0x08, 0x09, 0x0a,
+0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10};
+ private static final byte[] FOOTER_KEY_METADATA =
"footkey".getBytes(Charset.defaultCharset());
Review comment:
Fix it.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r463772543
##
File path:
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/example/ExampleParquetWriter.java
##
@@ -104,15 +105,19 @@ public Builder withExtraMetaData(Map
extraMetaData) {
return this;
}
+public Builder withWriteSupport(WriteSupport writeSupport) {
+ this.writeSupport = writeSupport;
+ return this;
+}
+
Review comment:
If we don't do that, we need to write Builder class to extend
ParquetWriter.Builder. It should be a thin wrapper. I think we can do it.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808: URL: https://github.com/apache/parquet-mr/pull/808#discussion_r463725057 ## File path: parquet-hadoop/src/test/java/org/apache/parquet/crypto/CryptoPropertiesFactoryTests/SchemaControlEncryptionTest.java ## @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.crypto.CryptoPropertiesFactoryTests; Review comment: Sounds good! This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [parquet-mr] shangxinli commented on a change in pull request #808: Parquet-1396: Cryptodata Interface for Schema Activation of Parquet E…
shangxinli commented on a change in pull request #808:
URL: https://github.com/apache/parquet-mr/pull/808#discussion_r463719447
##
File path: parquet-column/src/main/java/org/apache/parquet/schema/ExtType.java
##
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.schema;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class decorates the class 'Type' by adding a Map field 'metadata'.
+ *
+ * This decoration is needed to add metadata to each column without changing
existing class 'MessageType', which is used
+ * extensively. Here is the example usage to add column metadata to schema
with type of 'MessageType'.
+ *
+ * MessageType oldSchema = ...
+ * Map metadata = ...
+ * List newFields = new ArrayList();
+ * for (Type field = oldSchema.getFields()) {
+ * Type newField = new ExtType(field);
+ * newField.setMetadata(metadata);
+ * newFields.add(newField);
+ * }
+ * MessageType newSchema = new MessageType(oldSchema.getName(), newFields);
+ *
+ * The implementation is mostly following decoration pattern. Most of the
methods are just thin wrappers of existing
+ * implementation of PrimitiveType or GroupType.
+ */
+public class ExtType extends Type {
+ private Type type;
+ private Map metadata;
+
+ public ExtType(Type type) {
+super(type.getName(), type.getRepetition(), type.getOriginalType(),
type.getId());
+this.type = type;
+ }
+
+ public ExtType(Type type, String name) {
+super(name, type.getRepetition(), OriginalType.UINT_64, type.getId());
+this.type = new PrimitiveType(type.getRepetition(),
type.asPrimitiveType().getPrimitiveTypeName(), name);
+ }
+
+ public Type withId(int id) {
Review comment:
Sounds good
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
