Repository: orc Updated Branches: refs/heads/master d97654911 -> 6aa522415
ORC-256. Add unmasked ranges option for redact mask Fixes #184 Signed-off-by: Sandeep More <[email protected]> Signed-off-by: Owen O'Malley <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/orc/repo Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/6aa52241 Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/6aa52241 Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/6aa52241 Branch: refs/heads/master Commit: 6aa522415361e9f75a8bd2822721e76fb6f564e8 Parents: d976549 Author: Sandeep More <[email protected]> Authored: Sat Oct 28 10:41:41 2017 -0400 Committer: Owen O'Malley <[email protected]> Committed: Wed Dec 20 15:54:53 2017 -0800 ---------------------------------------------------------------------- .../apache/orc/impl/mask/RedactMaskFactory.java | 137 ++++++++++++++-- .../apache/orc/impl/mask/TestUnmaskRange.java | 162 +++++++++++++++++++ 2 files changed, 290 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/orc/blob/6aa52241/java/core/src/java/org/apache/orc/impl/mask/RedactMaskFactory.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/impl/mask/RedactMaskFactory.java b/java/core/src/java/org/apache/orc/impl/mask/RedactMaskFactory.java index 9574139..7e3e0a1 100644 --- a/java/core/src/java/org/apache/orc/impl/mask/RedactMaskFactory.java +++ b/java/core/src/java/org/apache/orc/impl/mask/RedactMaskFactory.java @@ -17,6 +17,7 @@ */ package org.apache.orc.impl.mask; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; @@ -25,16 +26,18 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.io.Text; -import org.apache.orc.TypeDescription; import org.apache.orc.DataMask; +import org.apache.orc.TypeDescription; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Calendar; +import java.util.Map; +import java.util.SortedMap; import java.util.TimeZone; +import java.util.TreeMap; import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; /** * Masking strategy that hides most string and numeric values based on unicode @@ -114,6 +117,9 @@ public class RedactMaskFactory extends MaskFactory { private final boolean maskDate; private final boolean maskTimestamp; + // index tuples that are not to be masked + private final SortedMap<Integer,Integer> unmaskIndexRanges = new TreeMap<>(); + public RedactMaskFactory(String... params) { ByteBuffer param = params.length < 1 ? ByteBuffer.allocate(0) : ByteBuffer.wrap(params[0].getBytes(StandardCharsets.UTF_8)); @@ -129,7 +135,7 @@ public class RedactMaskFactory extends MaskFactory { OTHER_NUMBER_REPLACEMENT = getNextCodepoint(param, DEFAULT_NUMBER_OTHER); OTHER_REPLACEMENT = getNextCodepoint(param, DEFAULT_OTHER); String[] timeParams; - if (params.length < 2) { + if (params.length < 2 || StringUtils.isBlank(params[1])) { timeParams = null; } else { timeParams = params[1].split("\\W+"); @@ -146,6 +152,16 @@ public class RedactMaskFactory extends MaskFactory { maskTimestamp = maskDate || (HOUR_REPLACEMENT != UNMASKED_DATE) || (MINUTE_REPLACEMENT != UNMASKED_DATE) || (SECOND_REPLACEMENT != UNMASKED_DATE); + + /* un-mask range */ + if(!(params.length < 3 || StringUtils.isBlank(params[2]))) { + String[] unmaskIndexes = params[2].split(","); + + for(int i=0; i < unmaskIndexes.length; i++ ) { + String[] pair = unmaskIndexes[i].trim().split(":"); + unmaskIndexRanges.put(Integer.parseInt(pair[0]), Integer.parseInt(pair[1])); + } + } } @Override @@ -451,6 +467,12 @@ public class RedactMaskFactory extends MaskFactory { * @return the masked value */ public long maskLong(long value) { + + /* check whether unmasking range provided */ + if (!unmaskIndexRanges.isEmpty()) { + return maskLongWithUnmasking(value); + } + long base; if (DIGIT_REPLACEMENT == 0) { return 0; @@ -521,6 +543,7 @@ public class RedactMaskFactory extends MaskFactory { } else { base *= 1_111_111_111_111_111_111L; } + return DIGIT_REPLACEMENT * base; } @@ -601,6 +624,12 @@ public class RedactMaskFactory extends MaskFactory { * @return the */ public double maskDouble(double value) { + + /* check whether unmasking range provided */ + if (!unmaskIndexRanges.isEmpty()) { + return maskDoubleWIthUnmasking(value); + } + double base; // It seems better to mask 0 to 9.99999 rather than 9.99999e-308. if (value == 0 || DIGIT_REPLACEMENT == 0) { @@ -684,8 +713,6 @@ public class RedactMaskFactory extends MaskFactory { return (int) (utcScratch.getTimeInMillis() / MILLIS_PER_DAY); } - private static final Pattern DIGIT_PATTERN = Pattern.compile("[0-9]"); - /** * Mask a decimal. * This is painfully slow because it converts to a string and then back to @@ -695,9 +722,7 @@ public class RedactMaskFactory extends MaskFactory { * @return the masked value. */ HiveDecimalWritable maskDecimal(HiveDecimalWritable source) { - String str = DIGIT_PATTERN.matcher(source.toString()). - replaceAll(Integer.toString(DIGIT_REPLACEMENT)); - return new HiveDecimalWritable(str); + return new HiveDecimalWritable(maskNumericString(source.toString())); } /** @@ -815,14 +840,20 @@ public class RedactMaskFactory extends MaskFactory { byte[] outputBuffer = target.getValPreallocatedBytes(); int outputOffset = target.getValPreallocatedStart(); int outputStart = outputOffset; + + int index = 0; while (sourceBytes.remaining() > 0) { int cp = Text.bytesToCodePoint(sourceBytes); // Find the replacement for the current character. int replacement = getReplacement(cp); - if (replacement == UNMASKED_CHAR) { + if (replacement == UNMASKED_CHAR || isIndexInUnmaskRange(index, source.length[row])) { replacement = cp; } + + // increment index + index++; + int len = getCodepointLength(replacement); // If the translation will overflow the buffer, we need to resize. @@ -854,4 +885,92 @@ public class RedactMaskFactory extends MaskFactory { } target.setValPreallocated(row, outputOffset - outputStart); } + + static final long OVERFLOW_REPLACEMENT = 111_111_111_111_111_111L; + + /** + * A function that masks longs when there are unmasked ranges. + * @param value the original value + * @return the masked value + */ + long maskLongWithUnmasking(long value) throws IndexOutOfBoundsException { + try { + return Long.parseLong(maskNumericString(Long.toString(value))); + } catch (NumberFormatException nfe) { + return OVERFLOW_REPLACEMENT * DIGIT_REPLACEMENT; + } + } + + /** + * A function that masks doubles when there are unmasked ranges. + * @param value original value + * @return masked value + */ + double maskDoubleWIthUnmasking(final double value) { + try { + return Double.parseDouble(maskNumericString(Double.toString(value))); + } catch (NumberFormatException nfe) { + return OVERFLOW_REPLACEMENT * DIGIT_REPLACEMENT; + } + } + + /** + * Mask the given stringified numeric value excluding the unmask range. + * Non-digit characters are passed through on the assumption they are + * markers (eg. one of ",.ef"). + * @param value the original value. + */ + String maskNumericString(final String value) { + StringBuilder result = new StringBuilder(); + final int length = value.codePointCount(0, value.length()); + for(int c=0; c < length; ++c) { + int cp = value.codePointAt(c); + if (isIndexInUnmaskRange(c, length) || + Character.getType(cp) != Character.DECIMAL_DIGIT_NUMBER) { + result.appendCodePoint(cp); + } else { + result.appendCodePoint(DIGIT_CP_REPLACEMENT); + } + } + return result.toString(); + } + + /** + * Given an index and length of a string + * find out whether it is in a given un-mask range. + * @param index the character point index + * @param length the length of the string in character points + * @return true if the index is in un-mask range else false. + */ + private boolean isIndexInUnmaskRange(final int index, final int length) { + + for(final Map.Entry<Integer, Integer> pair : unmaskIndexRanges.entrySet()) { + int start; + int end; + + if(pair.getKey() >= 0) { + // for positive indexes + start = pair.getKey(); + } else { + // for negative indexes + start = length + pair.getKey(); + } + + if(pair.getValue() >= 0) { + // for positive indexes + end = pair.getValue(); + } else { + // for negative indexes + end = length + pair.getValue(); + } + + // if the given index is in range + if(index >= start && index <= end ) { + return true; + } + + } + + return false; + } } http://git-wip-us.apache.org/repos/asf/orc/blob/6aa52241/java/core/src/test/org/apache/orc/impl/mask/TestUnmaskRange.java ---------------------------------------------------------------------- diff --git a/java/core/src/test/org/apache/orc/impl/mask/TestUnmaskRange.java b/java/core/src/test/org/apache/orc/impl/mask/TestUnmaskRange.java new file mode 100644 index 0000000..02269b6 --- /dev/null +++ b/java/core/src/test/org/apache/orc/impl/mask/TestUnmaskRange.java @@ -0,0 +1,162 @@ +package org.apache.orc.impl.mask; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; + +/** + * Test Unmask option + */ +public class TestUnmaskRange { + + public TestUnmaskRange() { + super(); + } + + /* Test for Long */ + @Test + public void testSimpleLongRangeMask() { + RedactMaskFactory mask = new RedactMaskFactory("9", "", "0:2"); + long result = mask.maskLong(123456); + assertEquals(123_999, result); + assertEquals(-129_999, mask.maskLong(-123456)); + + // negative index + mask = new RedactMaskFactory("9", "", "-3:-1"); + result = mask.maskLong(123456); + assertEquals(999_456, result); + assertEquals(-999_456, mask.maskLong(-123456)); + + // out of range mask, return the original mask + mask = new RedactMaskFactory("9", "", "7:10"); + result = mask.maskLong(123456); + assertEquals(999999, result); + + // if the masked value overflows, we get the overflow answer + result = mask.maskLong(1_234_567_890_123_456_789L); + assertEquals(999_999_999_999_999_999L, result); + } + + @Test + public void testDefaultRangeMask() { + RedactMaskFactory mask = new RedactMaskFactory("9", "", ""); + long result = mask.maskLong(123456); + assertEquals(999999, result); + + mask = new RedactMaskFactory("9"); + result = mask.maskLong(123456); + assertEquals(999999, result); + + } + + @Test + public void testCCRangeMask() { + long cc = 4716885592186382L; + long maskedCC = 4716_77777777_6382L; + // Range unmask for first 4 and last 4 of credit card number + final RedactMaskFactory mask = new RedactMaskFactory("Xx7", "", "0:3,-4:-1"); + long result = mask.maskLong(cc); + + assertEquals(String.valueOf(cc).length(), String.valueOf(result).length()); + assertEquals(4716_77777777_6382L, result); + } + + /* Tests for Double */ + @Test + public void testSimpleDoubleRangeMask() { + RedactMaskFactory mask = new RedactMaskFactory("Xx7", "", "0:2"); + assertEquals(1237.77, mask.maskDouble(1234.99), 0.000001); + assertEquals(12377.7, mask.maskDouble(12345.9), 0.000001); + + mask = new RedactMaskFactory("Xx7", "", "-3:-1"); + assertEquals(7774.9, mask.maskDouble(1234.9), 0.000001); + + } + + /* test for String */ + @Test + public void testStringRangeMask() { + + BytesColumnVector source = new BytesColumnVector(); + BytesColumnVector target = new BytesColumnVector(); + target.reset(); + + byte[] input = "Mary had 1 little lamb!!".getBytes(StandardCharsets.UTF_8); + source.setRef(0, input, 0, input.length); + + // Set a 4 byte chinese character (U+2070E), which is letter other + input = "\uD841\uDF0E".getBytes(StandardCharsets.UTF_8); + source.setRef(1, input, 0, input.length); + + RedactMaskFactory mask = new RedactMaskFactory("", "", "0:3, -5:-1"); + for(int r=0; r < 2; ++r) { + mask.maskString(source, r, target); + } + + assertEquals("Mary xxx 9 xxxxxx xamb!!", new String(target.vector[0], + target.start[0], target.length[0], StandardCharsets.UTF_8)); + assertEquals("\uD841\uDF0E", new String(target.vector[1], + target.start[1], target.length[1], StandardCharsets.UTF_8)); + + // test defaults, no-unmask range + mask = new RedactMaskFactory(); + for(int r=0; r < 2; ++r) { + mask.maskString(source, r, target); + } + + assertEquals("Xxxx xxx 9 xxxxxx xxxx..", new String(target.vector[0], + target.start[0], target.length[0], StandardCharsets.UTF_8)); + assertEquals("ª", new String(target.vector[1], + target.start[1], target.length[1], StandardCharsets.UTF_8)); + + + // test out of range string mask + mask = new RedactMaskFactory("", "", "-1:-5"); + for(int r=0; r < 2; ++r) { + mask.maskString(source, r, target); + } + + assertEquals("Xxxx xxx 9 xxxxxx xxxx..", new String(target.vector[0], + target.start[0], target.length[0], StandardCharsets.UTF_8)); + assertEquals("ª", new String(target.vector[1], + target.start[1], target.length[1], StandardCharsets.UTF_8)); + + } + + /* test for Decimal */ + @Test + public void testDecimalRangeMask() { + + RedactMaskFactory mask = new RedactMaskFactory("Xx7", "", "0:3"); + assertEquals(new HiveDecimalWritable("123477.777"), + mask.maskDecimal(new HiveDecimalWritable("123456.789"))); + + // try with a reverse index + mask = new RedactMaskFactory("Xx7", "", "-3:-1, 0:3"); + assertEquals(new HiveDecimalWritable("123477777.777654"), + mask.maskDecimal(new HiveDecimalWritable("123456789.987654"))); + + } + +}
