Repository: hive Updated Branches: refs/heads/master 89080f557 -> 6447f5cd5
HIVE-12954: NPE with str_to_map on null strings (Marta Kuczora, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6447f5cd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6447f5cd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6447f5cd Branch: refs/heads/master Commit: 6447f5cd57d193c6ceb6aaf141fb12f29ac53cd4 Parents: 89080f5 Author: Marta Kuczora <[email protected]> Authored: Thu Aug 11 12:28:03 2016 -0500 Committer: Sergio Pena <[email protected]> Committed: Thu Aug 11 12:28:50 2016 -0500 ---------------------------------------------------------------------- .../ql/udf/generic/GenericUDFStringToMap.java | 12 ++ .../udf/generic/TestGenericUDFStringToMap.java | 152 +++++++++++++++++++ 2 files changed, 164 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6447f5cd/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java index ed60fbf..093f2a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java @@ -83,11 +83,23 @@ public class GenericUDFStringToMap extends GenericUDF { public Object evaluate(DeferredObject[] arguments) throws HiveException { ret.clear(); String text = (String) soi_text.convert(arguments[0].get()); + if (text == null) { + return ret; + } + String delimiter1 = (soi_de1 == null) ? default_de1 : (String) soi_de1.convert(arguments[1].get()); String delimiter2 = (soi_de2 == null) ? default_de2 : (String) soi_de2.convert(arguments[2].get()); + if (delimiter1 == null) { + delimiter1 = default_de1; + } + + if (delimiter2 == null) { + delimiter2 = default_de2; + } + String[] keyValuePairs = text.split(delimiter1); for (String keyValuePair : keyValuePairs) { http://git-wip-us.apache.org/repos/asf/hive/blob/6447f5cd/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java new file mode 100644 index 0000000..1653936 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import static org.junit.Assert.assertTrue; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Test; + +public class TestGenericUDFStringToMap { + + @Test + public void testStringToMapWithCustomDelimiters() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDF(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("key1", "value1"); + expResult.put("key2", "value2"); + expResult.put("key3", "value3"); + runAndVerify("key1=value1;key2=value2;key3=value3", ";", "=", expResult, udf); + } + + @Test + public void testStringToMapWithDefaultDelimiters() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDFWithNoDelimiters(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("key1", "value1"); + expResult.put("key2", "value2"); + expResult.put("key3", "value3"); + runAndVerify("key1:value1,key2:value2,key3:value3", expResult, udf); + } + + @Test + public void testStringToMapWithNullDelimiters() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDF(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("key1", "value1"); + expResult.put("key2", "value2"); + expResult.put("key3", "value3"); + runAndVerify("key1:value1,key2:value2,key3:value3", null, null, expResult, udf); + } + + @Test + public void testStringToMapWithNullText() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDFWithNoDelimiters(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + runAndVerify(null, expResult, udf); + } + + @Test + public void testStringToMapWithEmptyText() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDFWithNoDelimiters(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("", null); + runAndVerify("", expResult, udf); + } + + @Test + public void testStringToMapNoKey() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDFWithNoDelimiters(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("", "value"); + runAndVerify(":value", expResult, udf); + } + + @Test + public void testStringToMapNoValue() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDFWithNoDelimiters(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("key", ""); + runAndVerify("key:", expResult, udf); + } + + @Test + public void testStringToMapNotMatchingDelimiter() throws HiveException { + GenericUDFStringToMap udf = new GenericUDFStringToMap(); + initGenericUDFWithNoDelimiters(udf); + Map<String, String> expResult = new LinkedHashMap<String, String>(); + expResult.put("key=value", null); + runAndVerify("key=value", expResult, udf); + } + + private void initGenericUDF(GenericUDFStringToMap udf) + throws UDFArgumentException { + + ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] arguments = { valueOI0, valueOI1, valueOI2 }; + udf.initialize(arguments); + } + + private void initGenericUDFWithNoDelimiters(GenericUDFStringToMap udf) + throws UDFArgumentException { + + ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] arguments = { valueOI0 }; + udf.initialize(arguments); + } + + private void runAndVerify(String text, String delimiter1, String delimiter2, + Map<String, String> expResult, GenericUDF udf) throws HiveException { + + DeferredObject valueObj0 = new DeferredJavaObject(text); + DeferredObject valueObj1 = new DeferredJavaObject(delimiter1); + DeferredObject valueObj2 = new DeferredJavaObject(delimiter2); + DeferredObject[] args = { valueObj0, valueObj1, valueObj2 }; + + @SuppressWarnings("unchecked") + LinkedHashMap<Object, Object> output = (LinkedHashMap<Object, Object>) udf.evaluate(args); + assertTrue("str_to_map() test", expResult.equals(output)); + } + + private void runAndVerify(String text, Map<String, String> expResult, + GenericUDF udf) throws HiveException { + + DeferredObject valueObj0 = new DeferredJavaObject(text); + DeferredObject[] args = { valueObj0 }; + @SuppressWarnings("unchecked") + LinkedHashMap<Object, Object> output = (LinkedHashMap<Object, Object>) udf.evaluate(args); + assertTrue("str_to_map() test", expResult.equals(output)); + } +} \ No newline at end of file
