Repository: incubator-ranger Updated Branches: refs/heads/master cba444bfd -> 2bd8581f9
RANGER-895: masking UDF impl update Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/2bd8581f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/2bd8581f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/2bd8581f Branch: refs/heads/master Commit: 2bd8581f9cebddb377110f29969ae95aff2f38a2 Parents: cba444b Author: Madhan Neethiraj <[email protected]> Authored: Tue Apr 26 08:29:08 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Apr 27 18:46:25 2016 -0700 ---------------------------------------------------------------------- .../authorization/hive/udf/RangerBaseUdf.java | 576 ++++++++++-------- .../authorization/hive/udf/RangerUdfMask.java | 601 +++++++++---------- .../hive/udf/RangerUdfMaskFirstN.java | 211 ++++++- .../hive/udf/RangerUdfMaskHash.java | 72 ++- .../hive/udf/RangerUdfMaskLastN.java | 176 +++++- .../hive/udf/RangerUdfMaskShowFirstN.java | 230 ++++++- .../hive/udf/RangerUdfMaskShowLastN.java | 180 +++++- 7 files changed, 1339 insertions(+), 707 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerBaseUdf.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerBaseUdf.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerBaseUdf.java index d2fbed9..89552da 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerBaseUdf.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerBaseUdf.java @@ -17,7 +17,6 @@ */ package org.apache.ranger.authorization.hive.udf; -// org.apache.hadoop.hive.ql.udf.generic. import org.apache.commons.logging.Log; @@ -40,371 +39,436 @@ import java.sql.Date; public abstract class RangerBaseUdf extends GenericUDF { - private static final Log LOG = LogFactory.getLog(RangerBaseUdf.class); + private static final Log LOG = LogFactory.getLog(RangerBaseUdf.class); - final protected RangerBaseUdf.RangerTransformer transformer; - protected RangerTransformerAdapter transformerAdapter = null; + final protected AbstractTransformer transformer; + final protected String displayName; + protected AbstractTransformerAdapter transformerAdapter = null; - protected RangerBaseUdf(RangerBaseUdf.RangerTransformer transformer) { - this.transformer = transformer; - } + protected RangerBaseUdf(AbstractTransformer transformer, String displayName) { + this.transformer = transformer; + this.displayName = displayName; + } - public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - LOG.debug("==> RangerBaseUdf.initialize()"); + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + LOG.debug("==> RangerBaseUdf.initialize()"); - checkArgPrimitive(arguments, 0); // first argument is the column to be transformed + checkArgPrimitive(arguments, 0); // first argument is the column to be transformed - PrimitiveObjectInspector columnType = ((PrimitiveObjectInspector) arguments[0]); + PrimitiveObjectInspector columnType = ((PrimitiveObjectInspector) arguments[0]); - transformer.init(arguments, 1); + transformer.init(arguments, 1); - transformerAdapter = RangerTransformerAdapter.getTransformerAdapter(columnType, transformer); + transformerAdapter = AbstractTransformerAdapter.getTransformerAdapter(columnType, transformer); - ObjectInspector ret = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(columnType.getPrimitiveCategory()); + ObjectInspector ret = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(columnType.getPrimitiveCategory()); - LOG.debug("<== RangerBaseUdf.initialize()"); + LOG.debug("<== RangerBaseUdf.initialize()"); - return ret; - } + return ret; + } - @Override - public Object evaluate(DeferredObject[] arguments) throws HiveException { - return transformerAdapter.getTransformedWritable(arguments[0]); - } + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object ret = transformerAdapter.getTransformedWritable(arguments[0]); - @Override - public String getDisplayString(String[] children) { - return getStandardDisplayString(getClass().getName(), children, ","); - } + return ret; + } - static abstract class RangerTransformer { - abstract void init(ObjectInspector[] arguments, int startIdx); - - abstract String transform(String value); - abstract Byte transform(Byte value); - abstract Short transform(Short value); - abstract Integer transform(Integer value); - abstract Long transform(Long value); - abstract Date transform(Date value); - } + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(displayName, children); + } } -abstract class RangerTransformerAdapter { - final RangerBaseUdf.RangerTransformer transformer; +/** + * Interface to be implemented by transformers which transform a given value according to its specification. + */ +abstract class AbstractTransformer { + /** + * Initialzie the transformer object + * @param arguments arguments given to GenericUDF.initialzie() + * @param startIdx index into array, from which the transformer should read values + */ + abstract void init(ObjectInspector[] arguments, int startIdx); + + /** + * Transform a String value + * @param value value to transform + * @return transformed value + */ + abstract String transform(String value); + + /** + * Transform a Byte value + * @param value value to transform + * @return transformed value + */ + abstract Byte transform(Byte value); + + /** + * Transform a Short value + * @param value value to transform + * @return transformed value + */ + abstract Short transform(Short value); + + /** + * Transform a Integer value + * @param value value to transform + * @return transformed value + */ + abstract Integer transform(Integer value); + + /** + * Transform a Long value + * @param value value to transform + * @return transformed value + */ + abstract Long transform(Long value); + + /** + * Transform a Date value + * @param value value to transform + * @return transformed value + */ + abstract Date transform(Date value); +} + +/** + * Interface to be implemented by datatype specific adapters that handle necessary conversion of the transformed value + * into appropriate Writable object, which GenericUDF.evaluate() is expected to return. + */ +abstract class AbstractTransformerAdapter { + final AbstractTransformer transformer; - RangerTransformerAdapter(RangerBaseUdf.RangerTransformer transformer) { - this.transformer = transformer; - } + AbstractTransformerAdapter(AbstractTransformer transformer) { + this.transformer = transformer; + } - abstract Object getTransformedWritable(DeferredObject value) throws HiveException; + abstract Object getTransformedWritable(DeferredObject value) throws HiveException; - static RangerTransformerAdapter getTransformerAdapter(PrimitiveObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - final RangerTransformerAdapter ret; + static AbstractTransformerAdapter getTransformerAdapter(PrimitiveObjectInspector columnType, AbstractTransformer transformer) { + final AbstractTransformerAdapter ret; - switch(columnType.getPrimitiveCategory()) { - case STRING: - ret = new StringTransformerAdapter((StringObjectInspector)columnType, transformer); - break; + switch(columnType.getPrimitiveCategory()) { + case STRING: + ret = new StringTransformerAdapter((StringObjectInspector)columnType, transformer); + break; - case VARCHAR: - ret = new HiveVarcharTransformerAdapter((HiveVarcharObjectInspector)columnType, transformer); - break; + case CHAR: + ret = new HiveCharTransformerAdapter((HiveCharObjectInspector)columnType, transformer); + break; - case CHAR: - ret = new HiveCharTransformerAdapter((HiveCharObjectInspector)columnType, transformer); - break; + case VARCHAR: + ret = new HiveVarcharTransformerAdapter((HiveVarcharObjectInspector)columnType, transformer); + break; - case BYTE: - ret = new ByteTransformerAdapter((ByteObjectInspector)columnType, transformer); - break; + case BYTE: + ret = new ByteTransformerAdapter((ByteObjectInspector)columnType, transformer); + break; - case SHORT: - ret = new ShortTransformerAdapter((ShortObjectInspector)columnType, transformer); - break; + case SHORT: + ret = new ShortTransformerAdapter((ShortObjectInspector)columnType, transformer); + break; - case INT: - ret = new IntegerTransformerAdapter((IntObjectInspector)columnType, transformer); - break; + case INT: + ret = new IntegerTransformerAdapter((IntObjectInspector)columnType, transformer); + break; - case LONG: - ret = new LongTransformerAdapter((LongObjectInspector)columnType, transformer); - break; + case LONG: + ret = new LongTransformerAdapter((LongObjectInspector)columnType, transformer); + break; - case DATE: - ret = new DateTransformerAdapter((DateObjectInspector)columnType, transformer); - break; + case DATE: + ret = new DateTransformerAdapter((DateObjectInspector)columnType, transformer); + break; - default: - ret = new NoTransformAdapter(columnType, transformer); - break; - } + default: + ret = new UnsupportedDatatypeTransformAdapter(columnType, transformer); + break; + } - return ret; - } + return ret; + } } -class ByteTransformerAdapter extends RangerTransformerAdapter { - final ByteObjectInspector columnType; - final ByteWritable writable; +class ByteTransformerAdapter extends AbstractTransformerAdapter { + final ByteObjectInspector columnType; + final ByteWritable writable; - public ByteTransformerAdapter(ByteObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new ByteWritable()); - } + public ByteTransformerAdapter(ByteObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new ByteWritable()); + } - public ByteTransformerAdapter(ByteObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, ByteWritable writable) { - super(transformer); + public ByteTransformerAdapter(ByteObjectInspector columnType, AbstractTransformer transformer, ByteWritable writable) { + super(transformer); - this.columnType = columnType; - this.writable = writable; - } + this.columnType = columnType; + this.writable = writable; + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - Byte value = (Byte)columnType.getPrimitiveJavaObject(object.get()); + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + Byte value = (Byte)columnType.getPrimitiveJavaObject(object.get()); - if(value != null) { - Byte transformedValue = transformer.transform(value); + if(value != null) { + Byte transformedValue = transformer.transform(value); - writable.set(transformedValue); + if(transformedValue != null) { + writable.set(transformedValue); - return writable; - } + return writable; + } + } - return null; - } + return null; + } } -class DateTransformerAdapter extends RangerTransformerAdapter { - final DateObjectInspector columnType; - final DateWritable writable; +class DateTransformerAdapter extends AbstractTransformerAdapter { + final DateObjectInspector columnType; + final DateWritable writable; - public DateTransformerAdapter(DateObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new DateWritable()); - } + public DateTransformerAdapter(DateObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new DateWritable()); + } - public DateTransformerAdapter(DateObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, DateWritable writable) { - super(transformer); + public DateTransformerAdapter(DateObjectInspector columnType, AbstractTransformer transformer, DateWritable writable) { + super(transformer); - this.columnType = columnType; - this.writable = writable; - } + this.columnType = columnType; + this.writable = writable; + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - Date value = columnType.getPrimitiveJavaObject(object.get()); + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + Date value = columnType.getPrimitiveJavaObject(object.get()); - if(value != null) { - Date transformedValue = transformer.transform(value); + if(value != null) { + Date transformedValue = transformer.transform(value); - writable.set(transformedValue); + if(transformedValue != null) { + writable.set(transformedValue); - return writable; - } + return writable; + } + } - return null; - } + return null; + } } -class HiveCharTransformerAdapter extends RangerTransformerAdapter { - final HiveCharObjectInspector columnType; - final HiveCharWritable writable; +class HiveCharTransformerAdapter extends AbstractTransformerAdapter { + final HiveCharObjectInspector columnType; + final HiveCharWritable writable; - public HiveCharTransformerAdapter(HiveCharObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new HiveCharWritable()); - } + public HiveCharTransformerAdapter(HiveCharObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new HiveCharWritable()); + } - public HiveCharTransformerAdapter(HiveCharObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, HiveCharWritable writable) { - super(transformer); + public HiveCharTransformerAdapter(HiveCharObjectInspector columnType, AbstractTransformer transformer, HiveCharWritable writable) { + super(transformer); - this.columnType = columnType; - this.writable = writable; - } + this.columnType = columnType; + this.writable = writable; + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - HiveChar value = columnType.getPrimitiveJavaObject(object.get()); + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + HiveChar value = columnType.getPrimitiveJavaObject(object.get()); - if(value != null) { - String transformedValue = transformer.transform(value.getValue()); + if(value != null) { + String transformedValue = transformer.transform(value.getValue()); - writable.set(transformedValue); + if(transformedValue != null) { + writable.set(transformedValue); - return writable; - } + return writable; + } + } - return null; - } + return null; + } } -class HiveVarcharTransformerAdapter extends RangerTransformerAdapter { - final HiveVarcharObjectInspector columnType; - final HiveVarcharWritable writable; +class HiveVarcharTransformerAdapter extends AbstractTransformerAdapter { + final HiveVarcharObjectInspector columnType; + final HiveVarcharWritable writable; - public HiveVarcharTransformerAdapter(HiveVarcharObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new HiveVarcharWritable()); - } + public HiveVarcharTransformerAdapter(HiveVarcharObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new HiveVarcharWritable()); + } - public HiveVarcharTransformerAdapter(HiveVarcharObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, HiveVarcharWritable writable) { - super(transformer); + public HiveVarcharTransformerAdapter(HiveVarcharObjectInspector columnType, AbstractTransformer transformer, HiveVarcharWritable writable) { + super(transformer); - this.columnType = columnType; - this.writable = writable; - } + this.columnType = columnType; + this.writable = writable; + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - HiveVarchar value = columnType.getPrimitiveJavaObject(object.get()); + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + HiveVarchar value = columnType.getPrimitiveJavaObject(object.get()); - if(value != null) { - String transformedValue = transformer.transform(value.getValue()); + if(value != null) { + String transformedValue = transformer.transform(value.getValue()); - writable.set(transformedValue); + if(transformedValue != null) { + writable.set(transformedValue); - return writable; - } + return writable; + } + } - return null; - } + return null; + } } -class IntegerTransformerAdapter extends RangerTransformerAdapter { - final IntObjectInspector columnType; - final IntWritable writable; +class IntegerTransformerAdapter extends AbstractTransformerAdapter { + final IntObjectInspector columnType; + final IntWritable writable; - public IntegerTransformerAdapter(IntObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new IntWritable()); - } + public IntegerTransformerAdapter(IntObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new IntWritable()); + } - public IntegerTransformerAdapter(IntObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, IntWritable writable) { - super(transformer); + public IntegerTransformerAdapter(IntObjectInspector columnType, AbstractTransformer transformer, IntWritable writable) { + super(transformer); - this.columnType = columnType; - this.writable = writable; - } + this.columnType = columnType; + this.writable = writable; + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - Integer value = (Integer)columnType.getPrimitiveJavaObject(object.get()); + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + Integer value = (Integer)columnType.getPrimitiveJavaObject(object.get()); - if(value != null) { - Integer transformedValue = transformer.transform(value); + if(value != null) { + Integer transformedValue = transformer.transform(value); - writable.set(transformedValue); + if(transformedValue != null) { + writable.set(transformedValue); - return writable; - } + return writable; + } + } - return null; - } + return null; + } } -class LongTransformerAdapter extends RangerTransformerAdapter { - final LongObjectInspector columnType; - final LongWritable writable; +class LongTransformerAdapter extends AbstractTransformerAdapter { + final LongObjectInspector columnType; + final LongWritable writable; - public LongTransformerAdapter(LongObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new LongWritable()); - } + public LongTransformerAdapter(LongObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new LongWritable()); + } - public LongTransformerAdapter(LongObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, LongWritable writable) { - super(transformer); + public LongTransformerAdapter(LongObjectInspector columnType, AbstractTransformer transformer, LongWritable writable) { + super(transformer); - this.columnType = columnType; - this.writable = writable; - } + this.columnType = columnType; + this.writable = writable; + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - Long value = (Long)columnType.getPrimitiveJavaObject(object.get()); + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + Long value = (Long)columnType.getPrimitiveJavaObject(object.get()); - if(value != null) { - Long transformedValue = transformer.transform(value); + if(value != null) { + Long transformedValue = transformer.transform(value); - writable.set(transformedValue); + if(transformedValue != null) { + writable.set(transformedValue); - return writable; - } + return writable; + } + } - return null; - } + return null; + } } -class NoTransformAdapter extends RangerTransformerAdapter { - final PrimitiveObjectInspector columnType; +class ShortTransformerAdapter extends AbstractTransformerAdapter { + final ShortObjectInspector columnType; + final ShortWritable writable; - public NoTransformAdapter(PrimitiveObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - super(transformer); + public ShortTransformerAdapter(ShortObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new ShortWritable()); + } - this.columnType = columnType; - } + public ShortTransformerAdapter(ShortObjectInspector columnType, AbstractTransformer transformer, ShortWritable writable) { + super(transformer); - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - return columnType.getPrimitiveWritableObject(object.get()); - } -} + this.columnType = columnType; + this.writable = writable; + } -class ShortTransformerAdapter extends RangerTransformerAdapter { - final ShortObjectInspector columnType; - final ShortWritable writable; + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + Short value = (Short)columnType.getPrimitiveJavaObject(object.get()); - public ShortTransformerAdapter(ShortObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new ShortWritable()); - } + if(value != null) { + Short transformedValue = transformer.transform(value); - public ShortTransformerAdapter(ShortObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, ShortWritable writable) { - super(transformer); + if(transformedValue != null) { + writable.set(transformedValue); - this.columnType = columnType; - this.writable = writable; - } + return writable; + } + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - Short value = (Short)columnType.getPrimitiveJavaObject(object.get()); + return null; + } +} - if(value != null) { - Short transformedValue = transformer.transform(value); +class StringTransformerAdapter extends AbstractTransformerAdapter { + final StringObjectInspector columnType; + final Text writable; - writable.set(transformedValue); + public StringTransformerAdapter(StringObjectInspector columnType, AbstractTransformer transformer) { + this(columnType, transformer, new Text()); + } - return writable; - } + public StringTransformerAdapter(StringObjectInspector columnType, AbstractTransformer transformer, Text writable) { + super(transformer); - return null; - } -} + this.columnType = columnType; + this.writable = writable; + } -class StringTransformerAdapter extends RangerTransformerAdapter { - final StringObjectInspector columnType; - final Text writable; + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + String value = columnType.getPrimitiveJavaObject(object.get()); - public StringTransformerAdapter(StringObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer) { - this(columnType, transformer, new Text()); - } + if(value != null) { + String transformedValue = transformer.transform(value); - public StringTransformerAdapter(StringObjectInspector columnType, RangerBaseUdf.RangerTransformer transformer, Text writable) { - super(transformer); + if(transformedValue != null) { + writable.set(transformedValue); - this.columnType = columnType; - this.writable = writable; - } + return writable; + } + } - @Override - public Object getTransformedWritable(DeferredObject object) throws HiveException { - String value = columnType.getPrimitiveJavaObject(object.get()); + return null; + } +} - if(value != null) { - String transformedValue = transformer.transform(value); +class UnsupportedDatatypeTransformAdapter extends AbstractTransformerAdapter { + final PrimitiveObjectInspector columnType; - writable.set(transformedValue); + public UnsupportedDatatypeTransformAdapter(PrimitiveObjectInspector columnType, AbstractTransformer transformer) { + super(transformer); - return writable; - } + this.columnType = columnType; + } - return null; - } + @Override + public Object getTransformedWritable(DeferredObject object) throws HiveException { + return null; + } } - - http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMask.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMask.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMask.java index 9d5b2ce..fd8ebba 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMask.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMask.java @@ -21,6 +21,7 @@ package org.apache.ranger.authorization.hive.udf; import java.sql.Date; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -29,311 +30,305 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; +@Description(name = "mask", + value = "masks the given value", + extended = "Examples:\n " + + " mask(ccn)\n " + + " mask(ccn, 'X', 'x', '0')\n " + + " mask(ccn, 'x', 'x', 'x')\n " + + "Arguments:\n " + + " mask(value, upperChar, lowerChar, digitChar, otherChar, numberChar, dayValue, monthValue, yearValue)\n " + + " value - value to mask. Supported types: TINYINT, SMALLINT, INT, BIGINT, STRING, VARCHAR, CHAR, DATE\n " + + " upperChar - character to replace upper-case characters with. Specify -1 to retain original character. Default value: 'X'\n " + + " lowerChar - character to replace lower-case characters with. Specify -1 to retain original character. Default value: 'x'\n " + + " digitChar - character to replace digit characters with. Specify -1 to retain original character. Default value: 'n'\n " + + " otherChar - character to replace all other characters with. Specify -1 to retain original character. Default value: -1\n " + + " numberChar - character to replace digits in a number with. Valid values: 0-9. Default value: '1'\n " + + " dayValue - value to replace day field in a date with. Specify -1 to retain original value. Valid values: 1-31. Default value: 1\n " + + " monthValue - value to replace month field in a date with. Specify -1 to retain original value. Valid values: 0-11. Default value: 0\n " + + " yearValue - value to replace year field in a date with. Specify -1 to retain original value. Default value: 0\n " + ) public class RangerUdfMask extends RangerBaseUdf { - public RangerUdfMask() { - super(new MaskTransformer()); - } + public static final String UDF_NAME = "mask"; + + public RangerUdfMask() { + super(new MaskTransformer(), UDF_NAME); + } } -class MaskTransformer extends RangerBaseUdf.RangerTransformer { - final static int MASKED_UPPERCASE = 'X'; - final static int MASKED_LOWERCASE = 'x'; - final static int MASKED_NUMBER = '1'; - final static int MASKED_OTHER_CHAR = -1; - final static int MASKED_DAY_COMPONENT_VAL = 1; - final static int MASKED_MONTH_COMPONENT_VAL = 0; - final static int MASKED_YEAR_COMPONENT_VAL = 0; - final static int UNMASKED_VAL = -1; - - int maskedUpperChar = MASKED_UPPERCASE; - int maskedLowerChar = MASKED_LOWERCASE; - int maskedDigitChar = MASKED_NUMBER; - int maskedOtherChar = MASKED_OTHER_CHAR; - int maskedNumber = MASKED_NUMBER; - int maskedDayValue = MASKED_DAY_COMPONENT_VAL; - int maskedMonthValue = MASKED_MONTH_COMPONENT_VAL; - int maskedYearValue = MASKED_YEAR_COMPONENT_VAL; - - public MaskTransformer() { - } - - @Override - public void init(ObjectInspector[] arguments, int startIdx) { - int idx = startIdx; - - maskedUpperChar = getCharArg(arguments, idx++, MASKED_UPPERCASE); - maskedLowerChar = getCharArg(arguments, idx++, MASKED_LOWERCASE); - maskedDigitChar = getCharArg(arguments, idx++, MASKED_NUMBER); - maskedOtherChar = getCharArg(arguments, idx++, MASKED_OTHER_CHAR); - maskedNumber = getCharArg(arguments, idx++, MASKED_NUMBER); - maskedDayValue = getIntArg(arguments, idx++, MASKED_DAY_COMPONENT_VAL); - maskedMonthValue = getIntArg(arguments, idx++, MASKED_MONTH_COMPONENT_VAL); - maskedYearValue = getIntArg(arguments, idx++, MASKED_YEAR_COMPONENT_VAL); - } - - @Override - String transform(String value) { - return maskString(value, 0, value.length()); - } - - @Override - Byte transform(Byte value) { - String strValue = value.toString(); - - return toByte(Long.parseLong(maskNumber(strValue, 0, strValue.length()))); - } - - @Override - Short transform(Short value) { - String strValue = value.toString(); - - return toShort(Long.parseLong(maskNumber(strValue, 0, strValue.length()))); - } - - @Override - Integer transform(Integer value) { - String strValue = value.toString(); - - return toInteger(Long.parseLong(maskNumber(strValue, 0, strValue.length()))); - } - - @Override - Long transform(Long value) { - String strValue = value.toString(); - - return Long.parseLong(maskNumber(strValue, 0, strValue.length())); - } - - @Override - Date transform(Date value) { - return maskDate(value); - } - - - String maskString(String val, int startIdx, int endIdx) { - StringBuilder strBuf = new StringBuilder(val.length()); - - if(startIdx < 0) { - startIdx = 0; - } - - if(endIdx < 0) { - endIdx = 0; - } - - if(startIdx > val.length()) { - startIdx = val.length(); - } - - if(endIdx > val.length()) { - endIdx = val.length(); - } - - for(int i = 0; i < startIdx; i++) { - strBuf.appendCodePoint(val.charAt(i)); - } - - for(int i = startIdx; i < endIdx; i++) { - int c = val.charAt(i); - - switch(Character.getType(c)) { - case Character.UPPERCASE_LETTER: - if(maskedUpperChar != UNMASKED_VAL) { - c = maskedUpperChar; - } - break; - - case Character.LOWERCASE_LETTER: - if(maskedLowerChar != UNMASKED_VAL) { - c = maskedLowerChar; - } - break; - - case Character.DECIMAL_DIGIT_NUMBER: - if(maskedDigitChar != UNMASKED_VAL) { - c = maskedDigitChar; - } - break; - - default: - if(maskedOtherChar != UNMASKED_VAL) { - c = maskedOtherChar; - } - break; - } - - strBuf.appendCodePoint(c); - } - - for(int i = endIdx; i < val.length(); i++) { - strBuf.appendCodePoint(val.charAt(i)); - } - - return strBuf.toString(); - } - - String maskNumber(String val, int startIdx, int endIdx) { - if(maskedNumber != UNMASKED_VAL) { - StringBuilder strBuf = new StringBuilder(val.length()); - - if (startIdx < 0) { - startIdx = 0; - } - - if (endIdx < 0) { - endIdx = 0; - } - - if (startIdx > val.length()) { - startIdx = val.length(); - } - - if (endIdx > val.length()) { - endIdx = val.length(); - } - - for (int i = 0; i < startIdx; i++) { - strBuf.appendCodePoint(val.charAt(i)); - } - - for (int i = startIdx; i < endIdx; i++) { - int c = val.charAt(i); - - switch (Character.getType(c)) { - case Character.DECIMAL_DIGIT_NUMBER: - c = maskedNumber; - break; - } - - strBuf.appendCodePoint(c); - } - - for (int i = endIdx; i < val.length(); i++) { - strBuf.appendCodePoint(val.charAt(i)); - } - - return strBuf.toString(); - } - - return val; - } - - Date maskDate(Date value) { - int year = maskedYearValue == UNMASKED_VAL ? value.getYear() : maskedYearValue; - int month = maskedMonthValue == UNMASKED_VAL ? value.getMonth() : maskedMonthValue; - int day = maskedDayValue == UNMASKED_VAL ? value.getDate() : maskedDayValue; - - return new Date(year, month, day); - } - - Byte toByte(long value) { - if(value < Byte.MIN_VALUE) { - return Byte.MIN_VALUE; - } else if(value > Byte.MAX_VALUE) { - return Byte.MAX_VALUE; - } else { - return (byte)value; - } - } - - Short toShort(long value) { - if(value < Short.MIN_VALUE) { - return Short.MIN_VALUE; - } else if(value > Short.MAX_VALUE) { - return Short.MAX_VALUE; - } else { - return (short)value; - } - } - - Integer toInteger(long value) { - if(value < Integer.MIN_VALUE) { - return Integer.MIN_VALUE; - } else if(value > Integer.MAX_VALUE) { - return Integer.MAX_VALUE; - } else { - return (int)value; - } - } - - int getCharArg(ObjectInspector[] arguments, int index, int defaultValue) { - int ret = defaultValue; - - ObjectInspector arg = (arguments != null && arguments.length > index) ? arguments[index] : null; - - if (arg != null) { - if(arg instanceof WritableConstantIntObjectInspector) { - IntWritable value = ((WritableConstantIntObjectInspector)arg).getWritableConstantValue(); - - if(value != null) { - ret = value.get(); - } - } else if(arg instanceof WritableConstantLongObjectInspector) { - LongWritable value = ((WritableConstantLongObjectInspector)arg).getWritableConstantValue(); - - if(value != null) { - ret = (int)value.get(); - } - } else if(arg instanceof WritableConstantShortObjectInspector) { - ShortWritable value = ((WritableConstantShortObjectInspector)arg).getWritableConstantValue(); - - if(value != null) { - ret = value.get(); - } - } else if(arg instanceof ConstantObjectInspector) { - Object value = ((ConstantObjectInspector) arg).getWritableConstantValue(); - - if (value != null) { - String strValue = value.toString(); - - if (strValue != null && strValue.length() > 0) { - ret = strValue.charAt(0); - } - } - } - } - - return ret; - } - - int getIntArg(ObjectInspector[] arguments, int index, int defaultValue) { - int ret = defaultValue; - - ObjectInspector arg = (arguments != null && arguments.length > index) ? arguments[index] : null; - - if (arg != null) { - if (arg instanceof WritableConstantIntObjectInspector) { - IntWritable value = ((WritableConstantIntObjectInspector) arg).getWritableConstantValue(); - - if (value != null) { - ret = value.get(); - } - } else if (arg instanceof WritableConstantLongObjectInspector) { - LongWritable value = ((WritableConstantLongObjectInspector) arg).getWritableConstantValue(); - - if (value != null) { - ret = (int) value.get(); - } - } else if (arg instanceof WritableConstantShortObjectInspector) { - ShortWritable value = ((WritableConstantShortObjectInspector) arg).getWritableConstantValue(); - - if (value != null) { - ret = value.get(); - } - } else if (arg instanceof ConstantObjectInspector) { - Object value = ((ConstantObjectInspector) arg).getWritableConstantValue(); - - if (value != null) { - String strValue = value.toString(); - - if (strValue != null && strValue.length() > 0) { - ret = Integer.parseInt(value.toString()); - } - } - } - } - - return ret; - } +class MaskTransformer extends AbstractTransformer { + final static int MASKED_UPPERCASE = 'X'; + final static int MASKED_LOWERCASE = 'x'; + final static int MASKED_DIGIT = 'n'; + final static int MASKED_OTHER_CHAR = -1; + final static int MASKED_NUMBER = 1; + final static int MASKED_DAY_COMPONENT_VAL = 1; + final static int MASKED_MONTH_COMPONENT_VAL = 0; + final static int MASKED_YEAR_COMPONENT_VAL = 0; + final static int UNMASKED_VAL = -1; + + int maskedUpperChar = MASKED_UPPERCASE; + int maskedLowerChar = MASKED_LOWERCASE; + int maskedDigitChar = MASKED_DIGIT; + int maskedOtherChar = MASKED_OTHER_CHAR; + int maskedNumber = MASKED_NUMBER; + int maskedDayValue = MASKED_DAY_COMPONENT_VAL; + int maskedMonthValue = MASKED_MONTH_COMPONENT_VAL; + int maskedYearValue = MASKED_YEAR_COMPONENT_VAL; + + public MaskTransformer() { + } + + @Override + public void init(ObjectInspector[] arguments, int startIdx) { + int idx = startIdx; + + maskedUpperChar = getCharArg(arguments, idx++, MASKED_UPPERCASE); + maskedLowerChar = getCharArg(arguments, idx++, MASKED_LOWERCASE); + maskedDigitChar = getCharArg(arguments, idx++, MASKED_DIGIT); + maskedOtherChar = getCharArg(arguments, idx++, MASKED_OTHER_CHAR); + maskedNumber = getIntArg(arguments, idx++, MASKED_NUMBER); + maskedDayValue = getIntArg(arguments, idx++, MASKED_DAY_COMPONENT_VAL); + maskedMonthValue = getIntArg(arguments, idx++, MASKED_MONTH_COMPONENT_VAL); + maskedYearValue = getIntArg(arguments, idx++, MASKED_YEAR_COMPONENT_VAL); + + if(maskedNumber < 0 || maskedNumber > 9) { + maskedNumber = MASKED_NUMBER; + } + + if(maskedDayValue < 1 || maskedDayValue > 31) { + maskedDayValue = MASKED_DAY_COMPONENT_VAL; + } + + if(maskedMonthValue < 0 || maskedMonthValue > 11) { + maskedMonthValue = MASKED_MONTH_COMPONENT_VAL; + } + } + + @Override + String transform(final String val) { + StringBuilder ret = new StringBuilder(val.length()); + + for(int i = 0; i < val.length(); i++) { + ret.appendCodePoint(transformChar(val.charAt(i))); + } + + return ret.toString(); + } + + @Override + Byte transform(final Byte value) { + byte val = value; + + if(value < 0) { + val *= -1; + } + + byte ret = 0; + int pos = 1; + while(val != 0) { + ret += maskedNumber * pos; + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Short transform(final Short value) { + short val = value; + + if(value < 0) { + val *= -1; + } + + short ret = 0; + int pos = 1; + while(val != 0) { + ret += maskedNumber * pos; + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Integer transform(final Integer value) { + int val = value; + + if(value < 0) { + val *= -1; + } + + int ret = 0; + int pos = 1; + while(val != 0) { + ret += maskedNumber * pos; + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Long transform(final Long value) { + long val = value; + + if(value < 0) { + val *= -1; + } + + long ret = 0; + long pos = 1; + for(int i = 0; val != 0; i++) { + ret += maskedNumber * pos; + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Date transform(final Date value) { + int year = maskedYearValue == UNMASKED_VAL ? value.getYear() : maskedYearValue; + int month = maskedMonthValue == UNMASKED_VAL ? value.getMonth() : maskedMonthValue; + int day = maskedDayValue == UNMASKED_VAL ? value.getDate() : maskedDayValue; + + return new Date(year, month, day); + } + + protected int transformChar(final int c) { + switch(Character.getType(c)) { + case Character.UPPERCASE_LETTER: + if(maskedUpperChar != UNMASKED_VAL) { + return maskedUpperChar; + } + break; + + case Character.LOWERCASE_LETTER: + if(maskedLowerChar != UNMASKED_VAL) { + return maskedLowerChar; + } + break; + + case Character.DECIMAL_DIGIT_NUMBER: + if(maskedDigitChar != UNMASKED_VAL) { + return maskedDigitChar; + } + break; + + default: + if(maskedOtherChar != UNMASKED_VAL) { + return maskedOtherChar; + } + break; + } + + return c; + } + + int getCharArg(ObjectInspector[] arguments, int index, int defaultValue) { + int ret = defaultValue; + + ObjectInspector arg = (arguments != null && arguments.length > index) ? arguments[index] : null; + + if (arg != null) { + if(arg instanceof WritableConstantIntObjectInspector) { + IntWritable value = ((WritableConstantIntObjectInspector)arg).getWritableConstantValue(); + + if(value != null) { + ret = value.get(); + } + } else if(arg instanceof WritableConstantLongObjectInspector) { + LongWritable value = ((WritableConstantLongObjectInspector)arg).getWritableConstantValue(); + + if(value != null) { + ret = (int)value.get(); + } + } else if(arg instanceof WritableConstantShortObjectInspector) { + ShortWritable value = ((WritableConstantShortObjectInspector)arg).getWritableConstantValue(); + + if(value != null) { + ret = value.get(); + } + } else if(arg instanceof ConstantObjectInspector) { + Object value = ((ConstantObjectInspector) arg).getWritableConstantValue(); + + if (value != null) { + String strValue = value.toString(); + + if (strValue != null && strValue.length() > 0) { + ret = strValue.charAt(0); + } + } + } + } + + return ret; + } + + int getIntArg(ObjectInspector[] arguments, int index, int defaultValue) { + int ret = defaultValue; + + ObjectInspector arg = (arguments != null && arguments.length > index) ? arguments[index] : null; + + if (arg != null) { + if (arg instanceof WritableConstantIntObjectInspector) { + IntWritable value = ((WritableConstantIntObjectInspector) arg).getWritableConstantValue(); + + if (value != null) { + ret = value.get(); + } + } else if (arg instanceof WritableConstantLongObjectInspector) { + LongWritable value = ((WritableConstantLongObjectInspector) arg).getWritableConstantValue(); + + if (value != null) { + ret = (int) value.get(); + } + } else if (arg instanceof WritableConstantShortObjectInspector) { + ShortWritable value = ((WritableConstantShortObjectInspector) arg).getWritableConstantValue(); + + if (value != null) { + ret = value.get(); + } + } else if (arg instanceof ConstantObjectInspector) { + Object value = ((ConstantObjectInspector) arg).getWritableConstantValue(); + + if (value != null) { + String strValue = value.toString(); + + if (strValue != null && strValue.length() > 0) { + ret = Integer.parseInt(value.toString()); + } + } + } + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskFirstN.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskFirstN.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskFirstN.java index 0c2605f..24c0988 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskFirstN.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskFirstN.java @@ -18,59 +18,212 @@ package org.apache.ranger.authorization.hive.udf; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +@Description(name = "mask_first_n", + value = "masks the first n characters of the value", + extended = "Examples:\n " + + " mask_first_n(ccn, 8)\n " + + " mask_first_n(ccn, 8, 'x', 'x', 'x')\n " + + "Arguments:\n " + + " mask(value, charCount, upperChar, lowerChar, digitChar, otherChar, numberChar)\n " + + " value - value to mask. Supported types: TINYINT, SMALLINT, INT, BIGINT, STRING, VARCHAR, CHAR\n " + + " charCount - number of characters. Default value: 4\n " + + " upperChar - character to replace upper-case characters with. Specify -1 to retain original character. Default value: 'X'\n " + + " lowerChar - character to replace lower-case characters with. Specify -1 to retain original character. Default value: 'x'\n " + + " digitChar - character to replace digit characters with. Specify -1 to retain original character. Default value: 'n'\n " + + " otherChar - character to replace all other characters with. Specify -1 to retain original character. Default value: -1\n " + + " numberChar - character to replace digits in a number with. Valid values: 0-9. Default value: '1'\n " + ) public class RangerUdfMaskFirstN extends RangerBaseUdf { - public RangerUdfMaskFirstN() { - super(new MaskFirstNTransformer()); - } + public static final String UDF_NAME = "mask_first_n"; + + public RangerUdfMaskFirstN() { + super(new MaskFirstNTransformer(), UDF_NAME); + } } class MaskFirstNTransformer extends MaskTransformer { - int charCount = 4; + int charCount = 4; + + public MaskFirstNTransformer() { + super(); + } + + @Override + public void init(ObjectInspector[] arguments, int argsStartIdx) { + super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed in this method below + + charCount = getIntArg(arguments, argsStartIdx, 4); + + if(charCount < 0) { + charCount = 0; + } + } + + @Override + String transform(final String value) { + final StringBuilder ret = new StringBuilder(value.length()); + final int endIdx = value.length() < charCount ? value.length() : charCount; + + for(int i = 0; i < endIdx; i++) { + ret.appendCodePoint(transformChar(value.charAt(i))); + } + + for(int i = endIdx; i < value.length(); i++) { + ret.appendCodePoint(value.charAt(i)); + } + + return ret.toString(); + } + + @Override + Byte transform(final Byte value) { + byte val = value; + + if(value < 0) { + val *= -1; + } + + // count number of digits in the value + int digitCount = 0; + for(byte v = val; v != 0; v /= 10) { + digitCount++; + } + + // number of digits to retain from the end + final int retainCount = digitCount < charCount ? 0 : (digitCount - charCount); + + byte ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= retainCount) { // mask this digit + ret += maskedNumber * pos; + } else { //retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - public MaskFirstNTransformer() { - super(); + if(value < 0) { + ret *= -1; } - @Override - public void init(ObjectInspector[] arguments, int argsStartIdx) { - super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed in this method below + return ret; + } + + @Override + Short transform(final Short value) { + short val = value; - charCount = getIntArg(arguments, argsStartIdx, 4); + if(value < 0) { + val *= -1; } - @Override - String transform(String value) { - return maskString(value, 0, charCount); + // count number of digits in the value + int digitCount = 0; + for(short v = val; v != 0; v /= 10) { + digitCount++; } - @Override - Byte transform(Byte value) { - String strValue = value.toString(); + // number of digits to retain from the end + final int retainCount = digitCount < charCount ? 0 : (digitCount - charCount); + + short ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= retainCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } - return toByte(Long.parseLong(maskNumber(strValue, 0, charCount))); + val /= 10; + pos *= 10; } - @Override - Short transform(Short value) { - String strValue = value.toString(); + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Integer transform(final Integer value) { + int val = value; - return toShort(Long.parseLong(maskNumber(strValue, 0, charCount))); + if(value < 0) { + val *= -1; } - @Override - Integer transform(Integer value) { - String strValue = value.toString(); + // count number of digits in the value + int digitCount = 0; + for(int v = val; v != 0; v /= 10) { + digitCount++; + } + + // number of digits to retain from the end + final int retainCount = digitCount < charCount ? 0 : (digitCount - charCount); + + int ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= retainCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - return toInteger(Long.parseLong(maskNumber(strValue, 0, charCount))); + if(value < 0) { + ret *= -1; } - @Override - Long transform(Long value) { - String strValue = value.toString(); + return ret; + } - return Long.parseLong(maskNumber(strValue, 0, charCount)); + @Override + Long transform(final Long value) { + long val = value; + + if(value < 0) { + val *= -1; + } + + // count number of digits in the value + int digitCount = 0; + for(long v = val; v != 0; v /= 10) { + digitCount++; + } + + // number of digits to retain from the end + final int retainCount = digitCount < charCount ? 0 : (digitCount - charCount); + + long ret = 0; + long pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= retainCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; } + + if(value < 0) { + ret *= -1; + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskHash.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskHash.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskHash.java index 4152949..430212b 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskHash.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskHash.java @@ -21,47 +21,57 @@ package org.apache.ranger.authorization.hive.udf; import java.sql.Date; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +@Description(name = "mask_hash", + value = "returns hash of the given value", + extended = "Examples:\n " + + " mask_hash(value)\n " + + "Arguments:\n " + + " value - value to mask. Supported types: STRING, VARCHAR, CHAR" + ) public class RangerUdfMaskHash extends RangerBaseUdf { - public RangerUdfMaskHash() { - super(new MaskHashTransformer()); - } + public static final String UDF_NAME = "mask_hash"; + + public RangerUdfMaskHash() { + super(new MaskHashTransformer(), UDF_NAME); + } } -class MaskHashTransformer extends RangerBaseUdf.RangerTransformer { - @Override - public void init(ObjectInspector[] arguments, int startIdx) { - } +class MaskHashTransformer extends AbstractTransformer { + @Override + public void init(ObjectInspector[] arguments, int startIdx) { + } - @Override - String transform(String value) { - return DigestUtils.md5Hex(value); - } + @Override + String transform(final String value) { + return DigestUtils.md5Hex(value); + } - @Override - Byte transform(Byte value) { - return value; - } + @Override + Byte transform(final Byte value) { + return null; + } - @Override - Short transform(Short value) { - return value; - } + @Override + Short transform(final Short value) { + return null; + } - @Override - Integer transform(Integer value) { - return value; - } + @Override + Integer transform(final Integer value) { + return null; + } - @Override - Long transform(Long value) { - return value; - } + @Override + Long transform(final Long value) { + return null; + } - @Override - Date transform(Date value) { - return value; - } + @Override + Date transform(final Date value) { + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskLastN.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskLastN.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskLastN.java index 97b24e4..554dc91 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskLastN.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskLastN.java @@ -18,60 +18,176 @@ package org.apache.ranger.authorization.hive.udf; -import org.apache.hadoop.hive.ql.udf.generic.*; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +@Description(name = "mask_last_n", + value = "masks the last n characters of the value", + extended = "Examples:\n " + + " mask_last_n(ccn, 8)\n " + + " mask_last_n(ccn, 8, 'x', 'x', 'x')\n " + + "Arguments:\n " + + " mask_last_n(value, charCount, upperChar, lowerChar, digitChar, otherChar, numberChar)\n " + + " value - value to mask. Supported types: TINYINT, SMALLINT, INT, BIGINT, STRING, VARCHAR, CHAR\n " + + " charCount - number of characters. Default value: 4\n " + + " upperChar - character to replace upper-case characters with. Specify -1 to retain original character. Default value: 'X'\n " + + " lowerChar - character to replace lower-case characters with. Specify -1 to retain original character. Default value: 'x'\n " + + " digitChar - character to replace digit characters with. Specify -1 to retain original character. Default value: 'n'\n " + + " otherChar - character to replace all other characters with. Specify -1 to retain original character. Default value: -1\n " + + " numberChar - character to replace digits in a number with. Valid values: 0-9. Default value: '1'\n " + ) public class RangerUdfMaskLastN extends RangerBaseUdf { - public RangerUdfMaskLastN() { - super(new MaskLastNTransformer()); - } + public static final String UDF_NAME = "mask_last_n"; + + public RangerUdfMaskLastN() { + super(new MaskLastNTransformer(), UDF_NAME); + } } class MaskLastNTransformer extends MaskTransformer { - int charCount = 4; + int charCount = 4; + + public MaskLastNTransformer() { + super(); + } + + @Override + public void init(ObjectInspector[] arguments, int argsStartIdx) { + super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed in this method below + + charCount = getIntArg(arguments, argsStartIdx, 4); + + if(charCount < 0) { + charCount = 0; + } + } + + @Override + String transform(final String value) { + final StringBuilder ret = new StringBuilder(value.length()); + final int startIdx = value.length() <= charCount ? 0 : (value.length() - charCount); + + for(int i = 0; i < startIdx; i++) { + ret.appendCodePoint(value.charAt(i)); + } + + for(int i = startIdx; i < value.length(); i++) { + ret.appendCodePoint(transformChar(value.charAt(i))); + } + + return ret.toString(); + } - public MaskLastNTransformer() { - super(); + @Override + Byte transform(final Byte value) { + byte val = value; + + if(value < 0) { + val *= -1; } - @Override - public void init(ObjectInspector[] arguments, int argsStartIdx) { - super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed in this method below + byte ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i < charCount) { // mask this digit + ret += maskedNumber * pos; + } else { //retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - charCount = getIntArg(arguments, argsStartIdx, 4); + if(value < 0) { + ret *= -1; } - @Override - String transform(String value) { - return maskString(value, value.length() - charCount, value.length()); + return ret; + } + + @Override + Short transform(final Short value) { + short val = value; + + if(value < 0) { + val *= -1; } - @Override - Byte transform(Byte value) { - String strValue = value.toString(); + short ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i < charCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - return toByte(Long.parseLong(maskNumber(strValue, strValue.length() - charCount, strValue.length()))); + if(value < 0) { + ret *= -1; } - @Override - Short transform(Short value) { - String strValue = value.toString(); + return ret; + } + + @Override + Integer transform(final Integer value) { + int val = value; - return toShort(Long.parseLong(maskNumber(strValue, strValue.length() - charCount, strValue.length()))); + if(value < 0) { + val *= -1; } - @Override - Integer transform(Integer value) { - String strValue = value.toString(); + int ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i < charCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - return toInteger(Long.parseLong(maskNumber(strValue, strValue.length() - charCount, strValue.length()))); + if(value < 0) { + ret *= -1; } - @Override - Long transform(Long value) { - String strValue = value.toString(); + return ret; + } - return Long.parseLong(maskNumber(strValue, strValue.length() - charCount, strValue.length())); + @Override + Long transform(final Long value) { + long val = value; + + if(value < 0) { + val *= -1; + } + + long ret = 0; + long pos = 1; + for(int i = 0; val != 0; i++) { + if(i < charCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; } + + if(value < 0) { + ret *= -1; + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowFirstN.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowFirstN.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowFirstN.java index 06a902c..24929b7 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowFirstN.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowFirstN.java @@ -18,59 +18,231 @@ package org.apache.ranger.authorization.hive.udf; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +@Description(name = "mask_show_first_n", + value = "masks all but first n characters of the value", + extended = "Examples:\n " + + " mask_show_first_n(ccn, 8)\n " + + " mask_show_first_n(ccn, 8, 'x', 'x', 'x')\n " + + "Arguments:\n " + + " mask_show_first_n(value, charCount, upperChar, lowerChar, digitChar, otherChar, numberChar)\n " + + " value - value to mask. Supported types: TINYINT, SMALLINT, INT, BIGINT, STRING, VARCHAR, CHAR\n " + + " charCount - number of characters. Default value: 4\n " + + " upperChar - character to replace upper-case characters with. Specify -1 to retain original character. Default value: 'X'\n " + + " lowerChar - character to replace lower-case characters with. Specify -1 to retain original character. Default value: 'x'\n " + + " digitChar - character to replace digit characters with. Specify -1 to retain original character. Default value: 'n'\n " + + " otherChar - character to replace all other characters with. Specify -1 to retain original character. Default value: -1\n " + + " numberChar - character to replace digits in a number with. Valid values: 0-9. Default value: '1'\n " + ) public class RangerUdfMaskShowFirstN extends RangerBaseUdf { - public RangerUdfMaskShowFirstN() { - super(new MaskShowFirstNTransformer()); - } + public static final String UDF_NAME = "mask_show_first_n"; + + public RangerUdfMaskShowFirstN() { + super(new MaskShowFirstNTransformer(), UDF_NAME); + } } class MaskShowFirstNTransformer extends MaskTransformer { - int charCount = 4; + int charCount = 4; + + public MaskShowFirstNTransformer() { + super(); + } + + @Override + public void init(ObjectInspector[] arguments, int argsStartIdx) { + super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed here + + charCount = getIntArg(arguments, argsStartIdx, 4); + + if(charCount < 0) { + charCount = 0; + } + } + + @Override + String transform(final String value) { + if(value.length() <= charCount) { + return value; + } + + final StringBuilder ret = new StringBuilder(value.length()); + + for(int i = 0; i < charCount; i++) { + ret.appendCodePoint(value.charAt(i)); + } + + for(int i = charCount; i < value.length(); i++) { + ret.appendCodePoint(transformChar(value.charAt(i))); + } + + return ret.toString(); + } + + @Override + Byte transform(final Byte value) { + byte val = value; + + if(value < 0) { + val *= -1; + } + + // count number of digits in the value + int digitCount = 0; + for(byte v = val; v != 0; v /= 10) { + digitCount++; + } + + // number of digits to mask from the end + final int maskCount = digitCount - charCount; + + if(maskCount <= 0) { + return value; + } + + byte ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i < maskCount) { // mask this digit + ret += (maskedNumber * pos); + } else { //retain this digit + ret += ((val % 10) * pos); + } + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Short transform(final Short value) { + short val = value; + + if(value < 0) { + val *= -1; + } + + // count number of digits in the value + int digitCount = 0; + for(short v = val; v != 0; v /= 10) { + digitCount++; + } + + // number of digits to mask from the end + final int maskCount = digitCount - charCount; + + if(maskCount <= 0) { + return value; + } - public MaskShowFirstNTransformer() { - super(); + short ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i < maskCount) { // mask this digit + ret += (maskedNumber * pos); + } else { // retain this digit + ret += ((val % 10) * pos); + } + + val /= 10; + pos *= 10; } - @Override - public void init(ObjectInspector[] arguments, int argsStartIdx) { - super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed here + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Integer transform(final Integer value) { + int val = value; - charCount = getIntArg(arguments, argsStartIdx, 4); + if(value < 0) { + val *= -1; } - @Override - String transform(String value) { - return maskString(value, charCount, value.length()); + // count number of digits in the value + int digitCount = 0; + for(int v = val; v != 0; v /= 10) { + digitCount++; + } + + // number of digits to mask from the end + final int maskCount = digitCount - charCount; + + if(maskCount <= 0) { + return value; } - @Override - Byte transform(Byte value) { - String strValue = value.toString(); + int ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i < maskCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += ((val % 10) * pos); + } + + val /= 10; + pos *= 10; + } - return toByte(Long.parseLong(maskNumber(strValue, charCount, strValue.length()))); + if(value < 0) { + ret *= -1; } - @Override - Short transform(Short value) { - String strValue = value.toString(); + return ret; + } + + @Override + Long transform(final Long value) { + long val = value; + + if(value < 0) { + val *= -1; + } - return toShort(Long.parseLong(maskNumber(strValue, charCount, strValue.length()))); + // count number of digits in the value + int digitCount = 0; + for(long v = val; v != 0; v /= 10) { + digitCount++; } - @Override - Integer transform(Integer value) { - String strValue = value.toString(); + // number of digits to mask from the end + final int maskCount = digitCount - charCount; - return toInteger(Long.parseLong(maskNumber(strValue, charCount, strValue.length()))); + if(maskCount <= 0) { + return value; } - @Override - Long transform(Long value) { - String strValue = value.toString(); + long ret = 0; + long pos = 1; + for(int i = 0; val != 0; i++) { + if(i < maskCount) { // mask this digit + ret += (maskedNumber * pos); + } else { // retain this digit + ret += ((val % 10) * pos); + } + + val /= 10; + pos *= 10; + } - return Long.parseLong(maskNumber(strValue, charCount, strValue.length())); + if(value < 0) { + ret *= -1; } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2bd8581f/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowLastN.java ---------------------------------------------------------------------- diff --git a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowLastN.java b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowLastN.java index be72097..d3db2af 100644 --- a/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowLastN.java +++ b/ranger-hive-utils/src/main/java/org/apache/ranger/authorization/hive/udf/RangerUdfMaskShowLastN.java @@ -18,59 +18,181 @@ package org.apache.ranger.authorization.hive.udf; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +@Description(name = "mask_show_last_n", + value = "masks all but last n characters of the value", + extended = "Examples:\n " + + " mask_show_last_n(ccn, 8)\n " + + " mask_show_last_n(ccn, 8, 'x', 'x', 'x')\n " + + "Arguments:\n " + + " mask_show_last_n(value, charCount, upperChar, lowerChar, digitChar, otherChar, numberChar)\n " + + " value - value to mask. Supported types: TINYINT, SMALLINT, INT, BIGINT, STRING, VARCHAR, CHAR\n " + + " charCount - number of characters. Default value: 4\n " + + " upperChar - character to replace upper-case characters with. Specify -1 to retain original character. Default value: 'X'\n " + + " lowerChar - character to replace lower-case characters with. Specify -1 to retain original character. Default value: 'x'\n " + + " digitChar - character to replace digit characters with. Specify -1 to retain original character. Default value: 'n'\n " + + " otherChar - character to replace all other characters with. Specify -1 to retain original character. Default value: -1\n " + + " numberChar - character to replace digits in a number with. Valid values: 0-9. Default value: '1'\n " + ) public class RangerUdfMaskShowLastN extends RangerBaseUdf { - public RangerUdfMaskShowLastN() { - super(new MaskShowLastNTransformer()); - } + public static final String UDF_NAME = "mask_show_last_n"; + + public RangerUdfMaskShowLastN() { + super(new MaskShowLastNTransformer(), UDF_NAME); + } } class MaskShowLastNTransformer extends MaskTransformer { - int charCount = 4; + int charCount = 4; + + public MaskShowLastNTransformer() { + super(); + } + + @Override + public void init(ObjectInspector[] arguments, int argsStartIdx) { + super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed in this method below + + charCount = getIntArg(arguments, argsStartIdx, 4); + + if(charCount < 0) { + charCount = 0; + } + } + - public MaskShowLastNTransformer() { - super(); + @Override + String transform(final String value) { + if(value.length() <= charCount) { + return value; } - @Override - public void init(ObjectInspector[] arguments, int argsStartIdx) { - super.init(arguments, argsStartIdx + 1); // first argument is charCount, which is consumed in this method below + final StringBuilder ret = new StringBuilder(value.length()); + final int endIdx = value.length() - charCount; - charCount = getIntArg(arguments, argsStartIdx, 4); + for(int i = 0; i < endIdx; i++) { + ret.appendCodePoint(transformChar(value.charAt(i))); } - @Override - String transform(String value) { - return maskString(value, 0, value.length() - charCount); + for(int i = endIdx; i < value.length(); i++) { + ret.appendCodePoint(value.charAt(i)); } - @Override - Byte transform(Byte value) { - String strValue = value.toString(); + return ret.toString(); + } - return toByte(Long.parseLong(maskNumber(strValue, 0, strValue.length() - charCount))); + @Override + Byte transform(final Byte value) { + byte val = value; + + if(value < 0) { + val *= -1; } - @Override - Short transform(Short value) { - String strValue = value.toString(); + byte ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= charCount) { // mask this digit + ret += maskedNumber * pos; + } else { //retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - return toShort(Long.parseLong(maskNumber(strValue, 0, strValue.length() - charCount))); + if(value < 0) { + ret *= -1; } - @Override - Integer transform(Integer value) { - String strValue = value.toString(); + return ret; + } - return toInteger(Long.parseLong(maskNumber(strValue, 0, strValue.length() - charCount))); + @Override + Short transform(final Short value) { + short val = value; + + if(value < 0) { + val *= -1; } - @Override - Long transform(Long value) { - String strValue = value.toString(); + short ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= charCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } - return Long.parseLong(maskNumber(strValue, 0, strValue.length() - charCount)); + if(value < 0) { + ret *= -1; } + + return ret; + } + + @Override + Integer transform(final Integer value) { + int val = value; + + if(value < 0) { + val *= -1; + } + + int ret = 0; + int pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= charCount) { // mask this digit + ret += maskedNumber * pos; + } else { // retain this digit + ret += (val % 10) * pos; + } + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } + + @Override + Long transform(final Long value) { + long val = value; + + if(value < 0) { + val *= -1; + } + + long ret = 0; + long pos = 1; + for(int i = 0; val != 0; i++) { + if(i >= charCount) { // mask this digit + ret += (maskedNumber * pos); + } else { // retain this digit + ret += ((val % 10) * pos); + } + + val /= 10; + pos *= 10; + } + + if(value < 0) { + ret *= -1; + } + + return ret; + } }
