Repository: incubator-hivemall Updated Branches: refs/heads/master 739cde516 -> e88d74fa4
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java b/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java index bb12ffa..9564cef 100644 --- a/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java +++ b/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java @@ -38,10 +38,8 @@ public class ConditionalEmitUDTFTest { ConditionalEmitUDTF udtf = new ConditionalEmitUDTF(); udtf.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaBooleanObjectInspector), - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaStringObjectInspector),}); + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),}); final List<Object> actual = new ArrayList<>(); udtf.setCollector(new Collector() { @@ -53,15 +51,15 @@ public class ConditionalEmitUDTFTest { } }); - udtf.process( - new Object[] {Arrays.asList(true, false, true), Arrays.asList("one", "two", "three")}); + udtf.process(new Object[] {Arrays.asList(true, false, true), + Arrays.asList("one", "two", "three")}); Assert.assertEquals(Arrays.asList("one", "three"), actual); actual.clear(); - udtf.process( - new Object[] {Arrays.asList(true, true, false), Arrays.asList("one", "two", "three")}); + udtf.process(new Object[] {Arrays.asList(true, true, false), + Arrays.asList("one", "two", "three")}); Assert.assertEquals(Arrays.asList("one", "two"), actual); udtf.close(); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java b/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java index 2291366..73ba47a 100644 --- a/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java +++ b/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java @@ -37,8 +37,7 @@ public class FirstElementUDFTest { public void test() throws IOException, HiveException { FirstElementUDF udf = new FirstElementUDF(); - udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); + udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject( WritableUtils.toWritableList(new double[] {0, 1, 2}))}; @@ -52,11 +51,10 @@ public class FirstElementUDFTest { public void testNull() throws IOException, HiveException { FirstElementUDF udf = new FirstElementUDF(); - udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); + udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); - DeferredObject[] args = new DeferredObject[] { - new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {}))}; + DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject( + WritableUtils.toWritableList(new double[] {}))}; Assert.assertNull(udf.evaluate(args)); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java b/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java index 177cfdd..38c7361 100644 --- a/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java +++ b/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java @@ -37,8 +37,7 @@ public class LastElementUDFTest { public void test() throws IOException, HiveException { LastElementUDF udf = new LastElementUDF(); - udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); + udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject( WritableUtils.toWritableList(new double[] {0, 1, 2}))}; @@ -52,11 +51,10 @@ public class LastElementUDFTest { public void testNull() throws IOException, HiveException { LastElementUDF udf = new LastElementUDF(); - udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); + udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}); - DeferredObject[] args = new DeferredObject[] { - new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {}))}; + DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject( + WritableUtils.toWritableList(new double[] {}))}; Assert.assertNull(udf.evaluate(args)); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java b/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java index 49848af..fc5572d 100644 --- a/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java +++ b/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java @@ -39,8 +39,8 @@ public class SelectKBestUDFTest { public void test() throws Exception { final SelectKBestUDF selectKBest = new SelectKBestUDF(); final int k = 2; - final double[] data = - new double[] {250.29999999999998, 170.90000000000003, 73.2, 12.199999999999996}; + final double[] data = new double[] {250.29999999999998, 170.90000000000003, 73.2, + 12.199999999999996}; final double[] importanceList = new double[] {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589}; @@ -50,10 +50,8 @@ public class SelectKBestUDFTest { new GenericUDF.DeferredJavaObject(k)}; selectKBest.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), ObjectInspectorUtils.getConstantObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, k)}); final List<DoubleWritable> resultObj = selectKBest.evaluate(dObjs); @@ -76,10 +74,8 @@ public class SelectKBestUDFTest { final SelectKBestUDF selectKBest = new SelectKBestUDF(); final int k = 2; selectKBest.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), ObjectInspectorUtils.getConstantObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, k)}); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java index 1aa8606..6f7d4cf 100644 --- a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java +++ b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java @@ -44,8 +44,8 @@ public class FromJsonUDFTest { ObjectInspector[] argOIs = new ObjectInspector[] { PrimitiveObjectInspectorFactory.writableStringObjectInspector, HiveUtils.getConstStringObjectInspector(types)}; - DeferredObject[] args = - new DeferredObject[] {new GenericUDF.DeferredJavaObject(new Text(json)), null}; + DeferredObject[] args = new DeferredObject[] { + new GenericUDF.DeferredJavaObject(new Text(json)), null}; udf.initialize(argOIs); Object result = udf.evaluate(args); @@ -67,8 +67,8 @@ public class FromJsonUDFTest { PrimitiveObjectInspectorFactory.writableStringObjectInspector, HiveUtils.getConstStringObjectInspector(types), HiveUtils.getConstStringObjectInspector("person")}; - DeferredObject[] args = - new DeferredObject[] {new GenericUDF.DeferredJavaObject(new Text(json)), null}; + DeferredObject[] args = new DeferredObject[] { + new GenericUDF.DeferredJavaObject(new Text(json)), null}; udf.initialize(argOIs); List<Object> result = (List<Object>) udf.evaluate(args); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java index d945cf5..4da6b9a 100644 --- a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java +++ b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java @@ -35,9 +35,7 @@ public class ToJsonUDFTest { public void testDoubleArray() throws Exception { ToJsonUDF udf = new ToJsonUDF(); - ObjectInspector[] argOIs = - new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}; + ObjectInspector[] argOIs = new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}; DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject( WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))}; http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java b/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java index 2f34d0e..9012c63 100644 --- a/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java +++ b/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java @@ -40,14 +40,12 @@ public class VectorAddUDFTest { VectorAddUDF udf = new VectorAddUDF(); udf.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableFloatObjectInspector)}); + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableFloatObjectInspector)}); DeferredObject[] args = new DeferredObject[] { - new GenericUDF.DeferredJavaObject( - WritableUtils.toWritableList(new double[] {1, 2, 3})), + new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {1, 2, + 3})), new GenericUDF.DeferredJavaObject( WritableUtils.toWritableList(new float[] {2, 3, 4}))}; @@ -64,16 +62,13 @@ public class VectorAddUDFTest { VectorAddUDF udf = new VectorAddUDF(); udf.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableLongObjectInspector), - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableIntObjectInspector)}); + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableIntObjectInspector)}); DeferredObject[] args = new DeferredObject[] { new GenericUDF.DeferredJavaObject( WritableUtils.toWritableList(new long[] {1, 2, 3})), - new GenericUDF.DeferredJavaObject( - WritableUtils.toWritableList(new int[] {2, 3, 4}))}; + new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new int[] {2, 3, 4}))}; List<?> actual = udf.evaluate(args); List<Long> expected = Arrays.asList(3L, 5L, 7L); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java b/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java index fb1e10f..6d7c05e 100644 --- a/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java +++ b/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java @@ -40,14 +40,12 @@ public class VectorDotUDFTest { VectorDotUDF udf = new VectorDotUDF(); udf.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableFloatObjectInspector)}); + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableFloatObjectInspector)}); DeferredObject[] args = new DeferredObject[] { - new GenericUDF.DeferredJavaObject( - WritableUtils.toWritableList(new double[] {1, 2, 3})), + new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {1, 2, + 3})), new GenericUDF.DeferredJavaObject( WritableUtils.toWritableList(new float[] {2, 3, 4}))}; @@ -64,14 +62,12 @@ public class VectorDotUDFTest { VectorDotUDF udf = new VectorDotUDF(); udf.initialize(new ObjectInspector[] { - ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), PrimitiveObjectInspectorFactory.writableFloatObjectInspector}); DeferredObject[] args = new DeferredObject[] { - new GenericUDF.DeferredJavaObject( - WritableUtils.toWritableList(new double[] {1, 2, 3})), - new GenericUDF.DeferredJavaObject(WritableUtils.val(2.f))}; + new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {1, 2, + 3})), new GenericUDF.DeferredJavaObject(WritableUtils.val(2.f))}; List<Double> actual = udf.evaluate(args); List<Double> expected = Arrays.asList(2.d, 4.d, 6.d); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/core/src/test/java/hivemall/utils/hadoop/JsonSerdeUtilsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/utils/hadoop/JsonSerdeUtilsTest.java b/core/src/test/java/hivemall/utils/hadoop/JsonSerdeUtilsTest.java index 5b971e7..556cc71 100644 --- a/core/src/test/java/hivemall/utils/hadoop/JsonSerdeUtilsTest.java +++ b/core/src/test/java/hivemall/utils/hadoop/JsonSerdeUtilsTest.java @@ -80,17 +80,18 @@ public class JsonSerdeUtilsTest { @Test public void testLooseJsonReadability() throws Exception { List<String> columnNames = Arrays.asList("s,k".split(",")); - List<TypeInfo> columnTypes = - TypeInfoUtils.getTypeInfosFromTypeString("struct<a:int,b:string>,int"); + List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString("struct<a:int,b:string>,int"); - Text jsonText1 = new Text("{ \"x\" : \"abc\" , " - + " \"t\" : { \"a\":\"1\", \"b\":\"2\", \"c\":[ { \"x\":2 , \"y\":3 } , { \"x\":3 , \"y\":2 }] } ," - + "\"s\" : { \"a\" : 2 , \"b\" : \"blah\", \"c\": \"woo\" } }"); + Text jsonText1 = new Text( + "{ \"x\" : \"abc\" , " + + " \"t\" : { \"a\":\"1\", \"b\":\"2\", \"c\":[ { \"x\":2 , \"y\":3 } , { \"x\":3 , \"y\":2 }] } ," + + "\"s\" : { \"a\" : 2 , \"b\" : \"blah\", \"c\": \"woo\" } }"); - Text jsonText2 = new Text("{ \"x\" : \"abc\" , " - + " \"t\" : { \"a\":\"1\", \"b\":\"2\", \"c\":[ { \"x\":2 , \"y\":3 } , { \"x\":3 , \"y\":2 }] } ," - + "\"s\" : { \"a\" : 2 , \"b\" : \"blah\", \"c\": \"woo\" } , " + "\"k\" : 113 " - + "}"); + Text jsonText2 = new Text( + "{ \"x\" : \"abc\" , " + + " \"t\" : { \"a\":\"1\", \"b\":\"2\", \"c\":[ { \"x\":2 , \"y\":3 } , { \"x\":3 , \"y\":2 }] } ," + + "\"s\" : { \"a\" : 2 , \"b\" : \"blah\", \"c\": \"woo\" } , " + + "\"k\" : 113 " + "}"); List<Object> expected1 = Arrays.<Object>asList(Arrays.asList(2, "blah"), null); List<Object> expected2 = Arrays.<Object>asList(Arrays.asList(2, "blah"), 113); @@ -104,8 +105,7 @@ public class JsonSerdeUtilsTest { @Test public void testMapValues() throws SerDeException { List<String> columnNames = Arrays.asList("a,b".split(",")); - List<TypeInfo> columnTypes = - TypeInfoUtils.getTypeInfosFromTypeString("array<string>,map<string,int>"); + List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString("array<string>,map<string,int>"); Text text1 = new Text("{ \"a\":[\"aaa\"],\"b\":{\"bbb\":1}} "); Text text2 = new Text("{\"a\":[\"yyy\"],\"b\":{\"zzz\":123}}"); @@ -193,22 +193,18 @@ public class JsonSerdeUtilsTest { DefaultHCatRecord r = new DefaultHCatRecord(rlist); - List<String> columnNames = - Arrays.asList("ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin".split(",")); - List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString( - "tinyint,smallint,int,bigint,double,float,string,string," - + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean," - + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>," - + "decimal(5,2),char(10),varchar(20),date,timestamp,binary"); + List<String> columnNames = Arrays.asList("ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin".split(",")); + List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString("tinyint,smallint,int,bigint,double,float,string,string," + + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean," + + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>," + + "decimal(5,2),char(10),varchar(20),date,timestamp,binary"); - StructTypeInfo rowTypeInfo = - (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); - HCatRecordObjectInspector objInspector = - HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo); + StructTypeInfo rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo( + columnNames, columnTypes); + HCatRecordObjectInspector objInspector = HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo); Text serialized = JsonSerdeUtils.serialize(r, objInspector, columnNames); - List<Object> deserialized = - JsonSerdeUtils.deserialize(serialized, columnNames, columnTypes); + List<Object> deserialized = JsonSerdeUtils.deserialize(serialized, columnNames, columnTypes); assertRecordEquals(rlist, deserialized); } @@ -240,22 +236,18 @@ public class JsonSerdeUtilsTest { DefaultHCatRecord r = new DefaultHCatRecord(nlist); - List<String> columnNames = - Arrays.asList("ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin".split(",")); - List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString( - "tinyint,smallint,int,bigint,double,float,string,string," - + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean," - + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>," - + "decimal(5,2),char(10),varchar(20),date,timestamp,binary"); + List<String> columnNames = Arrays.asList("ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin".split(",")); + List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString("tinyint,smallint,int,bigint,double,float,string,string," + + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean," + + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>," + + "decimal(5,2),char(10),varchar(20),date,timestamp,binary"); - StructTypeInfo rowTypeInfo = - (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); - HCatRecordObjectInspector objInspector = - HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo); + StructTypeInfo rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo( + columnNames, columnTypes); + HCatRecordObjectInspector objInspector = HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo); Text serialized = JsonSerdeUtils.serialize(r, objInspector, columnNames); - List<Object> deserialized = - JsonSerdeUtils.deserialize(serialized, columnNames, columnTypes); + List<Object> deserialized = JsonSerdeUtils.deserialize(serialized, columnNames, columnTypes); assertRecordEquals(nlist, deserialized); } @@ -266,11 +258,11 @@ public class JsonSerdeUtilsTest { TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("struct<name:string,age:int>"); List<Object> expected1 = Arrays.<Object>asList("makoto", 37); - List<Object> deserialized1 = - JsonSerdeUtils.deserialize(json1, Arrays.asList("person"), Arrays.asList(type1)); + List<Object> deserialized1 = JsonSerdeUtils.deserialize(json1, Arrays.asList("person"), + Arrays.asList(type1)); assertRecordEquals(expected1, deserialized1); - } + } @Test public void testTopLevelArray() throws Exception { @@ -337,15 +329,15 @@ public class JsonSerdeUtilsTest { int mySz = first.size(); int urSz = second.size(); if (mySz != urSz) { - throw new RuntimeException( - "#expected != #actual. #expected=" + mySz + ", #actual=" + urSz); + throw new RuntimeException("#expected != #actual. #expected=" + mySz + ", #actual=" + + urSz); } else { for (int i = 0; i < first.size(); i++) { int c = DataType.compare(first.get(i), second.get(i)); if (c != 0) { String msg = "first.get(" + i + "}='" + first.get(i) + "' second.get(" + i - + ")='" + second.get(i) + "' compared as " + c + "\n" + "Types 1st/2nd=" - + DataType.findType(first.get(i)) + "/" + + ")='" + second.get(i) + "' compared as " + c + "\n" + + "Types 1st/2nd=" + DataType.findType(first.get(i)) + "/" + DataType.findType(second.get(i)) + '\n' + "first='" + first.get(i) + "' second='" + second.get(i) + "'"; if (first.get(i) instanceof Date) { http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java ---------------------------------------------------------------------- diff --git a/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java b/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java index 384c317..89cf2c8 100644 --- a/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java +++ b/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java @@ -59,7 +59,8 @@ import org.apache.lucene.analysis.ja.dict.UserDictionary; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.util.CharArraySet; -@Description(name = "tokenize_ja", +@Description( + name = "tokenize_ja", value = "_FUNC_(String line [, const string mode = \"normal\", const array<string> stopWords, const array<string> stopTags, const array<string> userDict (or string userDictURL)])" + " - returns tokenized strings in array<string>") @UDFType(deterministic = true, stateful = false) @@ -83,21 +84,20 @@ public final class KuromojiUDF extends GenericUDF { public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { final int arglen = arguments.length; if (arglen < 1 || arglen > 5) { - throw new UDFArgumentException( - "Invalid number of arguments for `tokenize_ja`: " + arglen); + throw new UDFArgumentException("Invalid number of arguments for `tokenize_ja`: " + + arglen); } this._mode = (arglen >= 2) ? tokenizationMode(arguments[1]) : Mode.NORMAL; - this._stopWords = - (arglen >= 3) ? stopWords(arguments[2]) : JapaneseAnalyzer.getDefaultStopSet(); - this._stopTags = - (arglen >= 4) ? stopTags(arguments[3]) : JapaneseAnalyzer.getDefaultStopTags(); + this._stopWords = (arglen >= 3) ? stopWords(arguments[2]) + : JapaneseAnalyzer.getDefaultStopSet(); + this._stopTags = (arglen >= 4) ? stopTags(arguments[3]) + : JapaneseAnalyzer.getDefaultStopTags(); this._userDict = (arglen >= 5) ? userDictionary(arguments[4]) : null; this._analyzer = null; - return ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.writableStringObjectInspector); + return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector); } @Override @@ -265,17 +265,17 @@ public final class KuromojiUDF extends GenericUDF { final InputStream is; try { - is = IOUtils.decodeInputStream( - HttpUtils.getLimitedInputStream(conn, MAX_INPUT_STREAM_SIZE)); + is = IOUtils.decodeInputStream(HttpUtils.getLimitedInputStream(conn, + MAX_INPUT_STREAM_SIZE)); } catch (NullPointerException | IOException e) { throw new UDFArgumentException("Failed to get input stream from the connection: " + userDictURL + '\n' + ExceptionUtils.prettyPrintStackTrace(e)); } - CharsetDecoder decoder = - StandardCharsets.UTF_8.newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT); + CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter( + CodingErrorAction.REPORT); final Reader reader = new InputStreamReader(is, decoder); try { return UserDictionary.open(reader); // return null if empty http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/spark/common/src/main/java/hivemall/dataset/LogisticRegressionDataGeneratorUDTFWrapper.java ---------------------------------------------------------------------- diff --git a/spark/common/src/main/java/hivemall/dataset/LogisticRegressionDataGeneratorUDTFWrapper.java b/spark/common/src/main/java/hivemall/dataset/LogisticRegressionDataGeneratorUDTFWrapper.java index cf10ed7..a6d5468 100644 --- a/spark/common/src/main/java/hivemall/dataset/LogisticRegressionDataGeneratorUDTFWrapper.java +++ b/spark/common/src/main/java/hivemall/dataset/LogisticRegressionDataGeneratorUDTFWrapper.java @@ -41,8 +41,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @Description(name = "lr_datagen", value = "_FUNC_(options string) - Generates a logistic regression dataset") public final class LogisticRegressionDataGeneratorUDTFWrapper extends UDTFWithOptions { - private transient LogisticRegressionDataGeneratorUDTF udtf = - new LogisticRegressionDataGeneratorUDTF(); + private transient LogisticRegressionDataGeneratorUDTF udtf = new LogisticRegressionDataGeneratorUDTF(); @Override protected Options getOptions() { http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/e88d74fa/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java ---------------------------------------------------------------------- diff --git a/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java b/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java index b80f95a..d3e1bcb 100644 --- a/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java +++ b/xgboost/src/main/java/hivemall/xgboost/tools/XGBoostMulticlassPredictUDTF.java @@ -32,7 +32,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn import hivemall.utils.lang.Preconditions; -@Description(name = "xgboost_multiclass_predict", +@Description( + name = "xgboost_multiclass_predict", value = "_FUNC_(string rowid, string[] features, string model_id, array<byte> pred_model [, string options]) " + "- Returns a prediction result as (string rowid, string label, float probability)") public final class XGBoostMulticlassPredictUDTF extends hivemall.xgboost.XGBoostPredictUDTF {
