Repository: orc Updated Branches: refs/heads/master f90dee2da -> 74f20d5a3
ORC-250 - Create sha256 mask Fixes #208 Signed-off-by: Owen O'Malley <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/orc/repo Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/74f20d5a Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/74f20d5a Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/74f20d5a Branch: refs/heads/master Commit: 74f20d5a3b84b0702d48409f62c2795c9ff9627c Parents: f90dee2 Author: Sandeep More <[email protected]> Authored: Wed Jan 10 14:01:07 2018 -0500 Committer: Owen O'Malley <[email protected]> Committed: Thu Jan 18 13:33:21 2018 -0800 ---------------------------------------------------------------------- java/core/src/java/org/apache/orc/DataMask.java | 3 +- .../org/apache/orc/impl/mask/MaskProvider.java | 2 + .../apache/orc/impl/mask/SHA256MaskFactory.java | 288 +++++++++++++++++++ .../apache/orc/impl/mask/TestSHA256Mask.java | 262 +++++++++++++++++ 4 files changed, 554 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/orc/blob/74f20d5a/java/core/src/java/org/apache/orc/DataMask.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/DataMask.java b/java/core/src/java/org/apache/orc/DataMask.java index df6ae67..1abc031 100644 --- a/java/core/src/java/org/apache/orc/DataMask.java +++ b/java/core/src/java/org/apache/orc/DataMask.java @@ -38,7 +38,8 @@ public interface DataMask { */ enum Standard { NULLIFY("nullify"), - REDACT("redact"); + REDACT("redact"), + SHA256("sha256"); Standard(String name) { this.name = name; http://git-wip-us.apache.org/repos/asf/orc/blob/74f20d5a/java/core/src/java/org/apache/orc/impl/mask/MaskProvider.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/impl/mask/MaskProvider.java b/java/core/src/java/org/apache/orc/impl/mask/MaskProvider.java index 58f7b38..737a7bf 100644 --- a/java/core/src/java/org/apache/orc/impl/mask/MaskProvider.java +++ b/java/core/src/java/org/apache/orc/impl/mask/MaskProvider.java @@ -33,6 +33,8 @@ public class MaskProvider implements DataMask.Provider { return new NullifyMask(); } else if (name.equals(DataMask.Standard.REDACT.getName())) { return new RedactMaskFactory(params).build(schema); + } else if(name.equals(DataMask.Standard.SHA256.getName())) { + return new SHA256MaskFactory(params).build(schema); } return null; } http://git-wip-us.apache.org/repos/asf/orc/blob/74f20d5a/java/core/src/java/org/apache/orc/impl/mask/SHA256MaskFactory.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/impl/mask/SHA256MaskFactory.java b/java/core/src/java/org/apache/orc/impl/mask/SHA256MaskFactory.java new file mode 100644 index 0000000..062f957 --- /dev/null +++ b/java/core/src/java/org/apache/orc/impl/mask/SHA256MaskFactory.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.orc.impl.mask; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.orc.DataMask; +import org.apache.orc.TypeDescription; + +import javax.xml.bind.DatatypeConverter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +/** + * Masking strategy that masks String, Varchar, Char and Binary types + * as SHA 256 hash. + * <p> + * <b>For String type:</b><br/> + * All string type of any length will be converted to 64 character length + * SHA256 hash encoded in hexadecimal.<br/><br/> + * <p> + * <b>For Varchar type:</b><br/> + * For Varchar type, max-length property will be honored i.e. + * if the length is less than max-length then the SHA256 hash will be truncated + * to max-length. If max-length is greater than 64 then the output is the + * sha256 length, which is 64.<br/><br/> + * <p> + * <b>For Char type:</b><br/> + * For Char type, the length of mask will always be equal to specified + * max-length. If the given length (max-length) is less than SHA256 hash + * length (64) the mask will be truncated. + * If the given length (max-length) is greater than SHA256 hash length (64) + * then the mask will be padded by blank spaces.<br/><br/> + * <p> + * <b>For Binary type:</b><br/> + * All Binary type of any length will be converted to 32 byte length SHA256 + * hash.<br/> + */ +public class SHA256MaskFactory extends MaskFactory { + + final MessageDigest md; + + public SHA256MaskFactory(final String... params) { + super(); + try { + md = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + /** + * Mask a string by finding the character category of each character + * and replacing it with the matching literal. + * + * @param source the source column vector + * @param row the value index + * @param target the target column vector + * @param schema schema + */ + void maskString(final BytesColumnVector source, final int row, + final BytesColumnVector target, final TypeDescription schema) { + final ByteBuffer sourceBytes = ByteBuffer + .wrap(source.vector[row], source.start[row], source.length[row]); + + // take SHA-256 Hash and convert to HEX + byte[] hash = DatatypeConverter + .printHexBinary(md.digest(sourceBytes.array())) + .getBytes(StandardCharsets.UTF_8); + int targetLength = hash.length; + + switch (schema.getCategory()) { + /* For type varchar */ + case VARCHAR: { + /* truncate the hash if max length for varchar is less than hash length + * on the other hand if if the max length is more than hash length (64 + * bytes) we use the hash length (64 bytes) always. + */ + if (schema.getMaxLength() < hash.length) { + targetLength = schema.getMaxLength(); + } + break; + } + + case CHAR: { + /* for char the length is always constant */ + targetLength = schema.getMaxLength(); + break; + } + + default: { + targetLength = hash.length; + break; + } + } + + byte[] result; + + /* pad the hash with blank char if targetlength is greater than hash */ + if (targetLength > hash.length) { + result = new byte[targetLength]; + System.arraycopy(hash, 0, result, 0, hash.length); + Arrays.fill(result, hash.length, targetLength - 1, (byte) ' '); + } else { + result = new byte[targetLength]; + System.arraycopy(hash, 0, result, 0, targetLength); + } + + target.vector[row] = result; + target.start[row] = 0; + target.length[row] = targetLength; + } + + /** + * Helper function to mask binary data with it's SHA-256 hash. + * + * @param source + * @param row + * @param target + */ + void maskBinary(final BytesColumnVector source, final int row, + final BytesColumnVector target) { + + final ByteBuffer sourceBytes = ByteBuffer + .wrap(source.vector[row], source.start[row], source.length[row]); + + // take SHA-256 Hash and keep binary + byte[] hash = md.digest(sourceBytes.array()); + + int targetLength = hash.length; + + target.vector[row] = hash; + target.start[row] = 0; + target.length[row] = targetLength; + + } + + @Override + protected DataMask buildBinaryMask(TypeDescription schema) { + return new BinaryMask(); + } + + @Override + protected DataMask buildBooleanMask(TypeDescription schema) { + return new NullifyMask(); + } + + @Override + protected DataMask buildLongMask(TypeDescription schema) { + return new NullifyMask(); + } + + @Override + protected DataMask buildDecimalMask(TypeDescription schema) { + return new NullifyMask(); + } + + @Override + protected DataMask buildDoubleMask(TypeDescription schema) { + return new NullifyMask(); + } + + @Override + protected DataMask buildStringMask(final TypeDescription schema) { + return new StringMask(schema); + } + + @Override + protected DataMask buildDateMask(TypeDescription schema) { + return new NullifyMask(); + } + + @Override + protected DataMask buildTimestampMask(TypeDescription schema) { + return new NullifyMask(); + } + + /** + * Data mask for String, Varchar and Char types. + */ + class StringMask implements DataMask { + + final TypeDescription schema; + + /* create an instance */ + public StringMask(TypeDescription schema) { + super(); + this.schema = schema; + } + + /** + * Mask the given range of values + * + * @param original the original input data + * @param masked the masked output data + * @param start the first data element to mask + * @param length the number of data elements to mask + */ + @Override + public void maskData(final ColumnVector original, final ColumnVector masked, + final int start, final int length) { + + final BytesColumnVector target = (BytesColumnVector) masked; + final BytesColumnVector source = (BytesColumnVector) original; + + target.noNulls = original.noNulls; + target.isRepeating = original.isRepeating; + + if (original.isRepeating) { + target.isNull[0] = source.isNull[0]; + if (target.noNulls || !target.isNull[0]) { + maskString(source, 0, target, schema); + } + } else { + for (int r = start; r < start + length; ++r) { + target.isNull[r] = source.isNull[r]; + if (target.noNulls || !target.isNull[r]) { + maskString(source, r, target, schema); + } + } + } + + } + + } + + /** + * Mask for binary data + */ + class BinaryMask implements DataMask { + + /* create an instance */ + public BinaryMask() { + super(); + } + + /** + * Mask the given range of values + * + * @param original the original input data + * @param masked the masked output data + * @param start the first data element to mask + * @param length the number of data elements to mask + */ + @Override + public void maskData(final ColumnVector original, final ColumnVector masked, + final int start, final int length) { + + final BytesColumnVector target = (BytesColumnVector) masked; + final BytesColumnVector source = (BytesColumnVector) original; + + target.noNulls = original.noNulls; + target.isRepeating = original.isRepeating; + + if (original.isRepeating) { + target.isNull[0] = source.isNull[0]; + if (target.noNulls || !target.isNull[0]) { + maskBinary(source, 0, target); + } + } else { + for (int r = start; r < start + length; ++r) { + target.isNull[r] = source.isNull[r]; + if (target.noNulls || !target.isNull[r]) { + maskBinary(source, r, target); + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/orc/blob/74f20d5a/java/core/src/test/org/apache/orc/impl/mask/TestSHA256Mask.java ---------------------------------------------------------------------- diff --git a/java/core/src/test/org/apache/orc/impl/mask/TestSHA256Mask.java b/java/core/src/test/org/apache/orc/impl/mask/TestSHA256Mask.java new file mode 100644 index 0000000..f63b79c --- /dev/null +++ b/java/core/src/test/org/apache/orc/impl/mask/TestSHA256Mask.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.orc.impl.mask; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.orc.TypeDescription; +import org.junit.Test; + +import javax.xml.bind.DatatypeConverter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class TestSHA256Mask { + + final byte[] inputLong = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing " + + "elit. Curabitur quis vehicula ligula. In hac habitasse platea dictumst." + + " Curabitur mollis finibus erat fringilla vestibulum. In eu leo eget" + + " massa luctus convallis nec vitae ligula. Donec vitae diam convallis," + + " efficitur orci in, imperdiet turpis. In quis semper ex. Duis faucibus " + + "tellus vitae molestie convallis. Fusce fermentum vestibulum lacus " + + "vel malesuada. Pellentesque viverra odio a justo aliquet tempus.") + .getBytes(StandardCharsets.UTF_8); + + final byte[] input32 = "Every flight begins with a fall." + .getBytes(StandardCharsets.UTF_8); + + final byte[] inputShort = "\uD841\uDF0E".getBytes(StandardCharsets.UTF_8); + + final MessageDigest md; + + final byte[] expectedHash32 ; + final byte[] expectedHashShort ; + final byte[] expectedHashLong ; + + final byte[] expectedHash32_hex ; + final byte[] expectedHashShort_hex ; + final byte[] expectedHashLong_hex ; + + public TestSHA256Mask() { + super(); + try { + md = MessageDigest.getInstance("SHA-256"); + + expectedHash32 = md.digest(input32); + expectedHashShort = md.digest(inputShort); + expectedHashLong = md.digest(inputLong); + + expectedHash32_hex = DatatypeConverter.printHexBinary(expectedHash32).getBytes(StandardCharsets.UTF_8); + expectedHashShort_hex = DatatypeConverter.printHexBinary(expectedHashShort).getBytes(StandardCharsets.UTF_8); + expectedHashLong_hex = DatatypeConverter.printHexBinary(expectedHashLong).getBytes(StandardCharsets.UTF_8); + + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + /** + * Test to make sure that the output is always 64 bytes (equal to hash len) <br> + * This is because String type does not have bounds on length. + * + * @throws Exception + */ + @Test + public void testStringSHA256Masking() throws Exception { + final SHA256MaskFactory sha256Mask = new SHA256MaskFactory(); + final BytesColumnVector source = new BytesColumnVector(); + final BytesColumnVector target = new BytesColumnVector(); + + target.reset(); + + source.setRef(0, input32, 0, input32.length); + source.setRef(1, inputShort, 0, inputShort.length); + + for (int r = 0; r < 2; ++r) { + sha256Mask.maskString(source, r, target, TypeDescription.createString()); + } + + /* Make sure the the mask length is equal to 64 length of SHA-256 */ + assertEquals(64, target.length[0]); + assertEquals(64, target.length[1]); + + /* gather the results into an array to compare */ + byte[] reasultInput32 = new byte[target.length[0]]; + System.arraycopy(target.vector[0], target.start[0], reasultInput32, 0, target.length[0]); + byte[] reasultInputShort = new byte[target.length[1]]; + System.arraycopy(target.vector[1], target.start[1], reasultInputShort, 0, target.length[1]); + + /* prepare the expected byte[] to compare */ + final byte[] expected1 = new byte[target.length[0]]; + System.arraycopy(expectedHash32_hex, 0, expected1, 0, target.length[0]); + final byte[] expected2 = new byte[target.length[1]]; + System.arraycopy(expectedHashShort_hex, 0, expected2, 0, target.length[1]); + + /* Test the actual output. NOTE: our varchar has max length 32 */ + assertArrayEquals(expected1, reasultInput32); + assertArrayEquals(expected2, reasultInputShort); + + } + + /** + * Test to make sure that the length of input is equal to the output. <br> + * If input is shorter than the hash, truncate it. <br> + * If the input is larger than hash (64) pad it with blank space. <br> + * + * @throws Exception + */ + @Test + public void testChar256Masking() throws Exception { + final SHA256MaskFactory sha256Mask = new SHA256MaskFactory(); + final BytesColumnVector source = new BytesColumnVector(); + final BytesColumnVector target = new BytesColumnVector(); + + target.reset(); + + int[] length = new int[3]; + + length[0] = input32.length; + source.setRef(0, input32, 0, input32.length); + + length[1] = inputShort.length; + source.setRef(1, inputShort, 0, inputShort.length); + + length[2] = inputLong.length; + source.setRef(2, inputLong, 0, inputLong.length); + + for (int r = 0; r < 3; ++r) { + sha256Mask.maskString(source, r, target, + TypeDescription.createChar().withMaxLength(length[r])); + } + + /* Make sure the the mask length is equal to 64 length of SHA-256 */ + assertEquals(length[0], target.length[0]); + assertEquals(length[1], target.length[1]); + assertEquals(length[2], target.length[2]); + + } + + @Test + public void testVarChar256Masking() throws Exception { + final SHA256MaskFactory sha256Mask = new SHA256MaskFactory(); + final BytesColumnVector source = new BytesColumnVector(); + final BytesColumnVector target = new BytesColumnVector(); + + target.reset(); + + source.setRef(0, input32, 0, input32.length); + source.setRef(1, inputShort, 0, inputShort.length); + source.setRef(2, inputLong, 0, inputLong.length); + + for (int r = 0; r < 3; ++r) { + sha256Mask.maskString(source, r, target, + TypeDescription.createVarchar().withMaxLength(32)); + } + + /* gather the results into an array to compare */ + byte[] reasultInput32 = new byte[target.length[0]]; + System.arraycopy(target.vector[0], target.start[0], reasultInput32, 0, target.length[0]); + byte[] reasultInputShort = new byte[target.length[1]]; + System.arraycopy(target.vector[1], target.start[1], reasultInputShort, 0, target.length[1]); + byte[] reasultInputLong = new byte[target.length[2]]; + System.arraycopy(target.vector[2], target.start[2], reasultInputLong, 0, target.length[2]); + + + /* prepare the expected byte[] to compare */ + final byte[] expected1 = new byte[target.length[0]]; + System.arraycopy(expectedHash32_hex, 0, expected1, 0, target.length[0]); + final byte[] expected2 = new byte[target.length[1]]; + System.arraycopy(expectedHashShort_hex, 0, expected2, 0, target.length[1]); + final byte[] expected3 = new byte[target.length[2]]; + System.arraycopy(expectedHashLong_hex, 0, expected3, 0, target.length[2]); + + + // Hash is 64 in length greater than max len 32, so make sure output length is 32 + assertEquals(32, target.length[0]); + assertEquals(32, target.length[1]); + assertEquals(32, target.length[2]); + + /* Test the actual output. NOTE: our varchar has max length 32 */ + assertArrayEquals(expected1, reasultInput32); + assertArrayEquals(expected2, reasultInputShort); + assertArrayEquals(expected3, reasultInputLong); + + for (int r = 0; r < 3; ++r) { + sha256Mask.maskString(source, r, target, + TypeDescription.createVarchar().withMaxLength(100)); + } + + /* gather the results into an array to compare */ + reasultInput32 = new byte[target.length[0]]; + System.arraycopy(target.vector[0], target.start[0], reasultInput32, 0, target.length[0]); + reasultInputShort = new byte[target.length[1]]; + System.arraycopy(target.vector[1], target.start[1], reasultInputShort, 0, target.length[1]); + reasultInputLong = new byte[target.length[2]]; + System.arraycopy(target.vector[2], target.start[2], reasultInputLong, 0, target.length[2]); + + /* Hash is 64 in length, less than max len 100 so the outpur will always be 64 */ + assertEquals(64, target.length[0]); + assertEquals(64, target.length[1]); + assertEquals(64, target.length[2]); + + /* Test the actual output */ + assertArrayEquals(expectedHash32_hex, reasultInput32); + assertArrayEquals(expectedHashShort_hex, reasultInputShort); + assertArrayEquals(expectedHashLong_hex, reasultInputLong); + + } + + @Test + public void testBinary() { + + final SHA256MaskFactory sha256Mask = new SHA256MaskFactory(); + final BytesColumnVector source = new BytesColumnVector(); + final BytesColumnVector target = new BytesColumnVector(); + + target.reset(); + + source.setRef(0, input32, 0, input32.length); + source.setRef(1, inputShort, 0, inputShort.length); + source.setRef(2, inputLong, 0, inputLong.length); + + for (int r = 0; r < 3; ++r) { + sha256Mask.maskBinary(source, r, target); + } + + final byte[] reasultInput32 = ByteBuffer.wrap(target.vector[0], target.start[0], target.length[0]).array(); + final byte[] reasultInputShort = ByteBuffer.wrap(target.vector[1], target.start[1], target.length[1]).array(); + final byte[] reasultInputLong = ByteBuffer.wrap(target.vector[2], target.start[2], target.length[2]).array(); + + + /* Hash is 32 in length (length in binary), less than max len 100 so the output will always be 32 */ + assertEquals(32, target.length[0]); + assertEquals(32, target.length[1]); + assertEquals(32, target.length[2]); + + assertArrayEquals(expectedHash32, reasultInput32); + assertArrayEquals(expectedHashShort, reasultInputShort); + assertArrayEquals(expectedHashLong, reasultInputLong); + + + } + +}
