Repository: crunch Updated Branches: refs/heads/master ae0221f34 -> 2469348f6
CRUNCH-659: updated hive dependency to 2.1 Signed-off-by: Micah Whitacre <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/2469348f Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/2469348f Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/2469348f Branch: refs/heads/master Commit: 2469348f66b2e5ff45ed938753b35ddab83e6db4 Parents: ae0221f Author: Stephen Durfey <[email protected]> Authored: Thu Dec 7 09:55:56 2017 -0600 Committer: Micah Whitacre <[email protected]> Committed: Fri Dec 8 09:37:05 2017 -0600 ---------------------------------------------------------------------- .../crunch/types/orc/TupleObjectInspector.java | 37 +++++++++++--------- pom.xml | 4 +-- 2 files changed, 23 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/2469348f/crunch-hive/src/main/java/org/apache/crunch/types/orc/TupleObjectInspector.java ---------------------------------------------------------------------- diff --git a/crunch-hive/src/main/java/org/apache/crunch/types/orc/TupleObjectInspector.java b/crunch-hive/src/main/java/org/apache/crunch/types/orc/TupleObjectInspector.java index 9f5cd94..8673a7a 100644 --- a/crunch-hive/src/main/java/org/apache/crunch/types/orc/TupleObjectInspector.java +++ b/crunch-hive/src/main/java/org/apache/crunch/types/orc/TupleObjectInspector.java @@ -42,10 +42,10 @@ import org.apache.hadoop.io.BytesWritable; * */ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector { - + private TupleFactory<T> tupleFactory; private List<TupleField> fields; - + public TupleObjectInspector(TupleFactory<T> tupleFactory, PType... ptypes) { this.tupleFactory = tupleFactory; fields = new ArrayList<TupleField>(); @@ -54,23 +54,23 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector fields.add(field); } } - + static class TupleField implements StructField { - + private int index; private ObjectInspector oi; - + public TupleField(int index, PType<?> ptype) { this.index = index; oi = createObjectInspector(ptype); } - + private ObjectInspector createObjectInspector(PType<?> ptype) { Class typeClass = ptype.getTypeClass(); if (typeClass == Union.class || typeClass == Void.class) { throw new IllegalArgumentException(typeClass.getName() + " is not supported yet"); } - + ObjectInspector result; if (typeClass == ByteBuffer.class) { result = new ByteBufferObjectInspector(); @@ -103,10 +103,15 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector } @Override + public int getFieldID() { + return index; + } + + @Override public String getFieldComment() { return null; } - + } @Override @@ -130,7 +135,7 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector public Category getCategory() { return Category.STRUCT; } - + public T create(Object... values) { return tupleFactory.makeTuple(values); } @@ -149,7 +154,7 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector } return null; } - + @Override public Object getStructFieldData(Object data, StructField fieldRef) { TupleField field = (TupleField) fieldRef; @@ -165,14 +170,14 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector } return result; } - - + + static class ByteBufferObjectInspector extends AbstractPrimitiveJavaObjectInspector implements SettableBinaryObjectInspector { ByteBufferObjectInspector() { super(TypeInfoFactory.binaryTypeInfo); } - + @Override public ByteBuffer copyObject(Object o) { if (o == null) { @@ -184,7 +189,7 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector ByteBuffer duplicate = ByteBuffer.wrap(copiedBytes); return duplicate; } - + @Override public BytesWritable getPrimitiveWritableObject(Object o) { if (o == null) { @@ -195,7 +200,7 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector bw.set(buf.array(), buf.arrayOffset(), buf.limit()); return bw; } - + @Override public byte[] getPrimitiveJavaObject(Object o) { if (o == null) { @@ -227,7 +232,7 @@ public class TupleObjectInspector<T extends Tuple> extends StructObjectInspector return bw == null ? null : ByteBuffer.wrap(bw.getBytes()); } - + } } http://git-wip-us.apache.org/repos/asf/crunch/blob/2469348f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 919442b..9be4ba7 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ under the License. <commons-codec.version>1.4</commons-codec.version> <commons-cli.version>1.2</commons-cli.version> <avro.version>1.7.7</avro.version> - <hive.version>0.13.1</hive.version> + <hive.version>2.1.0</hive.version> <parquet.version>1.8.1</parquet.version> <javassist.version>3.16.1-GA</javassist.version> <jackson.databind.version>2.6.1</jackson.databind.version> @@ -99,7 +99,7 @@ under the License. <mockito.version>1.9.0</mockito.version> <pkg>org.apache.crunch</pkg> - <hadoop.version>2.5.2</hadoop.version> + <hadoop.version>2.6.0</hadoop.version> <hbase.version>1.0.0</hbase.version> <avro.classifier>hadoop2</avro.classifier>
