Author: gates
Date: Tue Feb 28 06:18:49 2012
New Revision: 1294477
URL: http://svn.apache.org/viewvc?rev=1294477&view=rev
Log:
HCATALOG-275 JSON SerDe issues
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Tue Feb 28 06:18:49 2012
@@ -52,6 +52,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-275 JSON SerDe issues (khorgath via gates)
+
HCAT-278 When outputSchema doesn't match table schema wrong columns are
returned to the user (gates)
HCAT-276 After merging in HCATALOG-237 related changes Pig scripts with more
than one store fail (daijy via gates)
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
(original)
+++
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
Tue Feb 28 06:18:49 2012
@@ -174,6 +174,10 @@ public class HCatRecordSerDe implements
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<Object> list = soi.getStructFieldsDataAsList(obj);
+
+ if (list == null){
+ return null;
+ }
List<Object> l = new ArrayList<Object>(fields.size());
@@ -182,7 +186,7 @@ public class HCatRecordSerDe implements
// Get the field objectInspector and the field object.
ObjectInspector foi = fields.get(i).getFieldObjectInspector();
- Object f = (list == null ? null : list.get(i));
+ Object f = list.get(i);
Object res = serializeField(f, foi);
l.add(i, res);
}
@@ -236,6 +240,10 @@ public class HCatRecordSerDe implements
private static List<?> serializeList(Object f, ListObjectInspector loi)
throws SerDeException {
List l = loi.getList(f);
+ if (l == null){
+ return null;
+ }
+
ObjectInspector eloi = loi.getListElementObjectInspector();
if (eloi.getCategory() == Category.PRIMITIVE){
List<Object> list = new ArrayList<Object>(l.size());
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
Tue Feb 28 06:18:49 2012
@@ -228,7 +228,11 @@ public class JsonSerDe implements SerDe
break;
case BOOLEAN:
String bval = (valueToken == JsonToken.VALUE_NULL)?null:p.getText();
- val = (bval.equalsIgnoreCase("true"));
+ if (bval != null){
+ val = Boolean.valueOf(bval);
+ } else {
+ val = null;
+ }
break;
case FLOAT:
val = (valueToken == JsonToken.VALUE_NULL)?null:p.getFloatValue();
@@ -242,6 +246,10 @@ public class JsonSerDe implements SerDe
case BINARY:
throw new IOException("JsonSerDe does not support BINARY type");
case ARRAY:
+ if (valueToken == JsonToken.VALUE_NULL){
+ val = null;
+ break;
+ }
if (valueToken != JsonToken.START_ARRAY){
throw new IOException("Start of Array expected");
}
@@ -252,6 +260,10 @@ public class JsonSerDe implements SerDe
val = arr;
break;
case MAP:
+ if (valueToken == JsonToken.VALUE_NULL){
+ val = null;
+ break;
+ }
if (valueToken != JsonToken.START_OBJECT){
throw new IOException("Start of Object expected");
}
@@ -272,6 +284,10 @@ public class JsonSerDe implements SerDe
val = map;
break;
case STRUCT:
+ if (valueToken == JsonToken.VALUE_NULL){
+ val = null;
+ break;
+ }
if (valueToken != JsonToken.START_OBJECT){
throw new IOException("Start of Object expected");
}
@@ -321,7 +337,28 @@ public class JsonSerDe implements SerDe
throws SerDeException {
StringBuilder sb = new StringBuilder();
try {
- buildJSONString(sb, obj, objInspector);
+
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
+ List<? extends StructField> structFields = soi.getAllStructFieldRefs();
+ assert (columnNames.size() == structFields.size());
+ if (obj == null) {
+ sb.append("null");
+ } else {
+ sb.append(SerDeUtils.LBRACE);
+ for (int i = 0; i < structFields.size(); i++) {
+ if (i > 0) {
+ sb.append(SerDeUtils.COMMA);
+ }
+ sb.append(SerDeUtils.QUOTE);
+ sb.append(columnNames.get(i));
+ sb.append(SerDeUtils.QUOTE);
+ sb.append(SerDeUtils.COLON);
+ buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)),
+ structFields.get(i).getFieldObjectInspector());
+ }
+ sb.append(SerDeUtils.RBRACE);
+ }
+
} catch (IOException e) {
LOG.warn("Error ["+ e.getMessage()+"] generating json text from object");
throw new SerDeException(e);
Modified:
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java
(original)
+++
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java
Tue Feb 28 06:18:49 2012
@@ -26,6 +26,7 @@ import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import junit.framework.Assert;
@@ -33,10 +34,10 @@ import junit.framework.TestCase;
public class TestJsonSerDe extends TestCase{
- public Map<Properties,HCatRecord> getData(){
- Map<Properties,HCatRecord> data = new HashMap<Properties,HCatRecord>();
+ public List<Pair<Properties,HCatRecord>> getData(){
+ List<Pair<Properties,HCatRecord>> data = new
ArrayList<Pair<Properties,HCatRecord>>();
- List<Object> rlist = new ArrayList<Object>(11);
+ List<Object> rlist = new ArrayList<Object>(13);
rlist.add(new Byte("123"));
rlist.add(new Short("456"));
rlist.add(new Integer(789));
@@ -80,6 +81,21 @@ public class TestJsonSerDe extends TestC
c1_1.add(i2);
c1.add(c1_1);
rlist.add(c1);
+
+ List<Object> nlist = new ArrayList<Object>(13);
+ nlist.add(null); // tinyint
+ nlist.add(null); // smallint
+ nlist.add(null); // int
+ nlist.add(null); // bigint
+ nlist.add(null); // double
+ nlist.add(null); // float
+ nlist.add(null); // string
+ nlist.add(null); // string
+ nlist.add(null); // struct
+ nlist.add(null); // array
+ nlist.add(null); // map
+ nlist.add(null); // bool
+ nlist.add(null); // complex
String typeString =
"tinyint,smallint,int,bigint,double,float,string,string,"
@@ -92,7 +108,8 @@ public class TestJsonSerDe extends TestC
// props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
// props.put(Constants.SERIALIZATION_FORMAT, "1");
- data.put(props, new DefaultHCatRecord(rlist));
+ data.add(new Pair(props, new DefaultHCatRecord(rlist)));
+ data.add(new Pair(props, new DefaultHCatRecord(nlist)));
return data;
}
@@ -100,9 +117,9 @@ public class TestJsonSerDe extends TestC
Configuration conf = new Configuration();
- for (Entry<Properties,HCatRecord> e : getData().entrySet()){
- Properties tblProps = e.getKey();
- HCatRecord r = e.getValue();
+ for (Pair<Properties,HCatRecord> e : getData()){
+ Properties tblProps = e.first;
+ HCatRecord r = e.second;
HCatRecordSerDe hrsd = new HCatRecordSerDe();
hrsd.initialize(conf, tblProps);
