Author: khorgath
Date: Thu Feb 16 14:45:20 2012
New Revision: 1245002
URL: http://svn.apache.org/viewvc?rev=1245002&view=rev
Log:
HCATALOG-241 Changes to HCatRecord to support switch from StorageDriver to
SerDe (khorgath) -- additional changes per toffer
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
Thu Feb 16 14:45:20 2012
@@ -525,30 +525,6 @@ public class HCatUtil {
+"<databasename>.<table name> or <table name>. Got " + tableName);
}
}
-
- public static boolean recordsEqual(HCatRecord first, HCatRecord second) {
- return (compareRecords(first,second) == 0);
- }
-
- public static int compareRecords(HCatRecord first, HCatRecord second) {
- return compareRecordContents(first.getAll(), second.getAll());
- }
-
- public static int compareRecordContents(List<Object> first, List<Object>
second) {
- int mySz = first.size();
- int urSz = second.size();
- if(mySz != urSz) {
- return mySz - urSz;
- } else {
- for (int i = 0; i < first.size(); i++) {
- int c = DataType.compare(first.get(i), second.get(i));
- if (c != 0) {
- return c;
- }
- }
- return 0;
- }
- }
public static ObjectInspector getObjectInspector(String serdeClassName,
Configuration conf, Properties tbl) throws Exception {
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
Thu Feb 16 14:45:20 2012
@@ -135,14 +135,4 @@ public abstract class HCatRecord impleme
set(fieldName,recordSchema,value);
}
-// @Override
-// public int compareTo(Object that) {
-// return HCatUtil.compareRecords(this,(HCatRecord)that);
-// }
-//
-// @Override
-// public boolean equals(Object other) {
-// return (compareTo(other) == 0);
-// }
-
}
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
(original)
+++
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
Thu Feb 16 14:45:20 2012
@@ -40,12 +40,12 @@ public class HCatRecordObjectInspector e
@Override
public Object getStructFieldData(Object data, StructField fieldRef) {
if (data == null){
- return null;
+ return new IllegalArgumentException("Data passed in to get field from
was null!");
}
int fieldID = ((MyField) fieldRef).getFieldID();
if (!(fieldID >= 0 && fieldID < fields.size())){
- throw new RuntimeException("Invalid field index ["+fieldID+"]");
+ throw new IllegalArgumentException("Invalid field index ["+fieldID+"]");
}
return ((HCatRecord) data).get(fieldID);
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
(original)
+++
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
Thu Feb 16 14:45:20 2012
@@ -30,29 +30,28 @@ public class LazyHCatRecord extends HCat
public Object get(int fieldNum) {
try {
StructField fref = soi.getAllStructFieldRefs().get(fieldNum);
- return deserialize(soi.getStructFieldData(o,
fref),fref.getFieldObjectInspector());
- } catch (Exception e) {
- throw new RuntimeException("SerDe Exception deserializing",e);
+ return HCatRecordSerDe.serializeField(
+ soi.getStructFieldData(o, fref),
+ fref.getFieldObjectInspector());
+ } catch (SerDeException e) {
+ throw new IllegalStateException("SerDe Exception deserializing",e);
}
}
- private Object deserialize(Object o, ObjectInspector objectInspector) throws
Exception {
- return HCatRecordSerDe.serializeField(o, objectInspector);
- }
@Override
public List<Object> getAll() {
List<Object> r = new ArrayList<Object>(this.size);
for (int i = 0; i < this.size; i++){
- r.set(i, get(i));
+ r.add(i, get(i));
}
return r;
}
@Override
public void set(int fieldNum, Object value) {
- throw new RuntimeException("not allowed to run set() on LazyHCatRecord");
+ throw new UnsupportedOperationException("not allowed to run set() on
LazyHCatRecord");
}
@Override
@@ -62,14 +61,14 @@ public class LazyHCatRecord extends HCat
@Override
public void readFields(DataInput in) throws IOException {
- throw new RuntimeException("LazyHCatRecord is intended to wrap"
+ throw new UnsupportedOperationException("LazyHCatRecord is intended to
wrap"
+ " an object/object inspector as a HCatRecord "
+ "- it does not need to be read from DataInput.");
}
@Override
public void write(DataOutput out) throws IOException {
- throw new RuntimeException("LazyHCatRecord is intended to wrap"
+ throw new UnsupportedOperationException("LazyHCatRecord is intended to
wrap"
+ " an object/object inspector as a HCatRecord "
+ "- it does not need to be written to a DataOutput.");
}
@@ -84,17 +83,17 @@ public class LazyHCatRecord extends HCat
@Override
public void set(String fieldName, HCatSchema recordSchema, Object value)
throws HCatException {
- throw new RuntimeException("not allowed to run set() on LazyHCatRecord");
+ throw new UnsupportedOperationException("not allowed to run set() on
LazyHCatRecord");
}
@Override
public void remove(int idx) throws HCatException {
- throw new RuntimeException("not allowed to run remove() on
LazyHCatRecord");
+ throw new UnsupportedOperationException("not allowed to run remove() on
LazyHCatRecord");
}
@Override
public void copy(HCatRecord r) throws HCatException {
- throw new RuntimeException("not allowed to run copy() on LazyHCatRecord");
+ throw new UnsupportedOperationException("not allowed to run copy() on
LazyHCatRecord");
}
public LazyHCatRecord(Object o, ObjectInspector oi) throws Exception{
Modified:
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java
(original)
+++
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java
Thu Feb 16 14:45:20 2012
@@ -19,6 +19,7 @@ package org.apache.hcatalog.data;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import java.util.Map.Entry;
import org.apache.commons.logging.Log;
@@ -87,4 +88,30 @@ public class HCatDataCheckUtil {
return src_values;
}
+
+ public static boolean recordsEqual(HCatRecord first, HCatRecord second) {
+ return (compareRecords(first,second) == 0);
+ }
+
+ public static int compareRecords(HCatRecord first, HCatRecord second) {
+ return compareRecordContents(first.getAll(), second.getAll());
+ }
+
+ public static int compareRecordContents(List<Object> first, List<Object>
second) {
+ int mySz = first.size();
+ int urSz = second.size();
+ if(mySz != urSz) {
+ return mySz - urSz;
+ } else {
+ for (int i = 0; i < first.size(); i++) {
+ int c = DataType.compare(first.get(i), second.get(i));
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+ }
+
+
}
Modified:
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
(original)
+++
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
Thu Feb 16 14:45:20 2012
@@ -64,7 +64,7 @@ public class TestDefaultHCatRecord exten
for(int i =0; i < recs.length; i++){
HCatRecord rec = new DefaultHCatRecord();
rec.readFields(inpStream);
- Assert.assertTrue(HCatUtil.recordsEqual(recs[i],rec));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(recs[i],rec));
}
Assert.assertEquals(fInStream.available(), 0);
@@ -74,13 +74,13 @@ public class TestDefaultHCatRecord exten
public void testCompareTo() {
HCatRecord[] recs = getHCatRecords();
- Assert.assertTrue(HCatUtil.compareRecords(recs[0],recs[1]) == 0);
+ Assert.assertTrue(HCatDataCheckUtil.compareRecords(recs[0],recs[1]) == 0);
}
public void testEqualsObject() {
HCatRecord[] recs = getHCatRecords();
- Assert.assertTrue(HCatUtil.recordsEqual(recs[0],recs[1]));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(recs[0],recs[1]));
}
private HCatRecord[] getHCatRecords(){
Modified:
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
(original)
+++
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
Thu Feb 16 14:45:20 2012
@@ -116,7 +116,7 @@ public class TestHCatRecordSerDe extends
System.out.println("ONE:"+s.toString());
HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
- Assert.assertTrue(HCatUtil.recordsEqual(r,r2));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,r2));
// If it went through correctly, then s is also a HCatRecord,
// and also equal to the above, and a deepcopy, and this holds
@@ -124,8 +124,8 @@ public class TestHCatRecordSerDe extends
Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
System.out.println("TWO:"+s2.toString());
- Assert.assertTrue(HCatUtil.recordsEqual(r,(HCatRecord)s));
- Assert.assertTrue(HCatUtil.recordsEqual(r,(HCatRecord)s2));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,(HCatRecord)s));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,(HCatRecord)s2));
// serialize using another serde, and read out that object repr.
LazySimpleSerDe testSD = new LazySimpleSerDe();
@@ -147,11 +147,6 @@ public class TestHCatRecordSerDe extends
LazyHCatRecord s6 = new LazyHCatRecord(s4,hrsd.getObjectInspector());
System.out.println("SIX:"+s6.toString());
- DelimitedJSONSerDe jsde = new DelimitedJSONSerDe();
- jsde.initialize(conf,tblProps);
-
- Writable s7 = jsde.serialize(s6, hrsd.getObjectInspector());
- System.out.println("SEVEN:"+s7);
}
}
Modified:
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
(original)
+++
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
Thu Feb 16 14:45:20 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.DefaultHCatRecord;
+import org.apache.hcatalog.data.HCatDataCheckUtil;
import org.apache.hcatalog.data.HCatRecord;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.hcatalog.rcfile.RCFileInputDriver;
@@ -119,7 +120,7 @@ public class TestRCFileInputStorageDrive
Assert.assertEquals(bytesArr[j], w);
HCatRecord t = sd.convertToHCatRecord(null,w);
Assert.assertEquals(8, t.size());
- Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(t,tuples[j]));
}
}
@@ -147,7 +148,7 @@ public class TestRCFileInputStorageDrive
Assert.assertEquals(w.size(), 8);
HCatRecord t = sd.convertToHCatRecord(null,w);
Assert.assertEquals(5, t.size());
- Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(t,tuples[j]));
}
assertFalse(rr.nextKeyValue());
}
@@ -179,7 +180,7 @@ public class TestRCFileInputStorageDrive
Assert.assertEquals(w.size(), 8);
HCatRecord t = sd.convertToHCatRecord(null,w);
Assert.assertEquals(7, t.size());
- Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
+ Assert.assertTrue(HCatDataCheckUtil.recordsEqual(t,tuples[j]));
}
assertFalse(rr.nextKeyValue());
}
