Repository: orc Updated Branches: refs/heads/master 7a4fe255e -> 545fe3712
ORC-53. Make the complex types Comparable so they can be put as keys to the OrcMap. (omalley) Fixes #29 Signed-off-by: Owen O'Malley <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/orc/repo Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/545fe371 Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/545fe371 Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/545fe371 Branch: refs/heads/master Commit: 545fe37128111a39b24faf40ae0851069317d879 Parents: 7a4fe25 Author: Owen O'Malley <[email protected]> Authored: Thu May 26 08:17:29 2016 -0700 Committer: Owen O'Malley <[email protected]> Committed: Wed Jun 1 19:51:57 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/orc/TypeDescription.java | 64 ++++++++- .../org/apache/orc/impl/SchemaEvolution.java | 6 +- java/core/src/test/resources/log4j.properties | 5 + .../src/java/org/apache/orc/mapred/OrcList.java | 36 ++++- .../src/java/org/apache/orc/mapred/OrcMap.java | 60 +++++++- .../org/apache/orc/mapred/OrcRecordReader.java | 12 +- .../java/org/apache/orc/mapred/OrcStruct.java | 66 +++++++-- .../org/apache/orc/mapred/OrcTimestamp.java | 6 +- .../java/org/apache/orc/mapred/OrcUnion.java | 27 +++- .../test/org/apache/orc/mapred/TestOrcList.java | 45 ++++++ .../test/org/apache/orc/mapred/TestOrcMap.java | 138 ++++++++++++++++++- .../org/apache/orc/mapred/TestOrcStruct.java | 44 ++++++ .../org/apache/orc/mapred/TestOrcUnion.java | 33 +++++ .../src/test/resources/log4j.properties | 5 + java/pom.xml | 1 + java/storage-api/pom.xml | 12 ++ 16 files changed, 515 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/core/src/java/org/apache/orc/TypeDescription.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/TypeDescription.java b/java/core/src/java/org/apache/orc/TypeDescription.java index d4c66d1..da9fe49 100644 --- a/java/core/src/java/org/apache/orc/TypeDescription.java +++ b/java/core/src/java/org/apache/orc/TypeDescription.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -37,12 +38,61 @@ import java.util.List; /** * This is the description of the types in an ORC file. */ -public class TypeDescription { +public class TypeDescription + implements Comparable<TypeDescription>, Serializable { private static final int MAX_PRECISION = 38; private static final int MAX_SCALE = 38; private static final int DEFAULT_PRECISION = 38; private static final int DEFAULT_SCALE = 10; private static final int DEFAULT_LENGTH = 256; + + @Override + public int compareTo(TypeDescription other) { + if (this == other) { + return 0; + } else if (other == null) { + return -1; + } else { + int result = category.compareTo(other.category); + if (result == 0) { + switch (category) { + case CHAR: + case VARCHAR: + return maxLength - other.maxLength; + case DECIMAL: + if (precision != other.precision) { + return precision - other.precision; + } + return scale - other.scale; + case UNION: + case LIST: + case MAP: + if (children.size() != other.children.size()) { + return children.size() - other.children.size(); + } + for(int c=0; result == 0 && c < children.size(); ++c) { + result = children.get(c).compareTo(other.children.get(c)); + } + break; + case STRUCT: + if (children.size() != other.children.size()) { + return children.size() - other.children.size(); + } + for(int c=0; result == 0 && c < children.size(); ++c) { + result = fieldNames.get(c).compareTo(other.fieldNames.get(c)); + if (result == 0) { + result = children.get(c).compareTo(other.children.get(c)); + } + } + break; + default: + // PASS + } + } + return result; + } + } + public enum Category { BOOLEAN("boolean", true), BYTE("tinyint", true), @@ -463,12 +513,18 @@ public class TypeDescription { @Override public int hashCode() { - return getId(); + long result = category.ordinal() * 4241 + maxLength + precision * 13 + scale; + if (children != null) { + for(TypeDescription child: children) { + result = result * 6959 + child.hashCode(); + } + } + return (int) result; } @Override public boolean equals(Object other) { - if (other == null || other.getClass() != TypeDescription.class) { + if (other == null || !(other instanceof TypeDescription)) { return false; } if (other == this) { @@ -476,8 +532,6 @@ public class TypeDescription { } TypeDescription castOther = (TypeDescription) other; if (category != castOther.category || - getId() != castOther.getId() || - getMaximumId() != castOther.getMaximumId() || maxLength != castOther.maxLength || scale != castOther.scale || precision != castOther.precision) { http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java index 2c80aaa..3e4e47b 100644 --- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java +++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java @@ -33,7 +33,7 @@ import org.apache.orc.TypeDescription; * has been schema evolution. */ public class SchemaEvolution { - private final Map<TypeDescription, TypeDescription> readerToFile; + private final Map<Integer, TypeDescription> readerToFile; private final boolean[] included; private final TypeDescription readerSchema; private static final Log LOG = LogFactory.getLog(SchemaEvolution.class); @@ -70,7 +70,7 @@ public class SchemaEvolution { result = null; } } else { - result = readerToFile.get(readerType); + result = readerToFile.get(readerType.getId()); } return result; } @@ -143,7 +143,7 @@ public class SchemaEvolution { isOk = ConvertTreeReaderFactory.canConvert(fileType, readerType); } if (isOk) { - readerToFile.put(readerType, fileType); + readerToFile.put(readerType.getId(), fileType); } else { throw new IOException( String.format( http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/core/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/java/core/src/test/resources/log4j.properties b/java/core/src/test/resources/log4j.properties new file mode 100644 index 0000000..67e071d --- /dev/null +++ b/java/core/src/test/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO,stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%p\t%d{ISO8601}\t%r\t%c\t[%t]\t%m%n http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java index 2b94207..b950671 100644 --- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java +++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java @@ -17,7 +17,7 @@ */ package org.apache.orc.mapred; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; import org.apache.orc.TypeDescription; import java.io.DataInput; @@ -30,8 +30,8 @@ import java.util.Iterator; * An ArrayList implementation that implements Writable. * @param <E> the element type, which must be Writable */ -public class OrcList<E extends Writable> - extends ArrayList<E> implements Writable { +public class OrcList<E extends WritableComparable> + extends ArrayList<E> implements WritableComparable<OrcList<E>> { private final TypeDescription childSchema; public OrcList(TypeDescription schema) { @@ -71,4 +71,34 @@ public class OrcList<E extends Writable> } } } + + @Override + public int compareTo(OrcList<E> other) { + if (other == null) { + return -1; + } + int result = childSchema.compareTo(other.childSchema); + if (result != 0) { + return result; + } + int ourSize = size(); + int otherSize = other.size(); + for(int e=0; e < ourSize && e < otherSize; ++e) { + E ours = get(e); + E theirs = other.get(e); + if (ours == null) { + if (theirs != null) { + return 1; + } + } else if (theirs == null) { + return -1; + } else { + int val = ours.compareTo(theirs); + if (val != 0) { + return val; + } + } + } + return ourSize - otherSize; + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java index db961fc..cf47827 100644 --- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java +++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java @@ -17,24 +17,24 @@ */ package org.apache.orc.mapred; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; import org.apache.orc.TypeDescription; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; import java.util.Iterator; import java.util.Map; import java.util.TreeMap; /** * A TreeMap implementation that implements Writable. - * @param <K> the key type, which must be Writable - * @param <V> the value type, which must be Writable + * @param <K> the key type, which must be WritableComparable + * @param <V> the value type, which must be WritableComparable */ -public final class OrcMap<K extends Writable, V extends Writable> - extends TreeMap<K, V> implements Writable { +public final class OrcMap<K extends WritableComparable, + V extends WritableComparable> + extends TreeMap<K, V> implements WritableComparable<OrcMap<K,V>> { private final TypeDescription keySchema; private final TypeDescription valueSchema; @@ -83,4 +83,52 @@ public final class OrcMap<K extends Writable, V extends Writable> put(key, value); } } + + @Override + public int compareTo(OrcMap<K,V> other) { + if (other == null) { + return -1; + } + int result = keySchema.compareTo(other.keySchema); + if (result != 0) { + return result; + } + result = valueSchema.compareTo(other.valueSchema); + if (result != 0) { + return result; + } + Iterator<Map.Entry<K,V>> ourItr = entrySet().iterator(); + Iterator<Map.Entry<K,V>> theirItr = other.entrySet().iterator(); + while (ourItr.hasNext() && theirItr.hasNext()) { + Map.Entry<K,V> ourItem = ourItr.next(); + Map.Entry<K,V> theirItem = theirItr.next(); + K ourKey = ourItem.getKey(); + K theirKey = theirItem.getKey(); + int val = ourKey.compareTo(theirKey); + if (val != 0) { + return val; + } + Comparable<V> ourValue = ourItem.getValue(); + V theirValue = theirItem.getValue(); + if (ourValue == null) { + if (theirValue != null) { + return 1; + } + } else if (theirValue == null) { + return -1; + } else { + val = ourItem.getValue().compareTo(theirItem.getValue()); + if (val != 0) { + return val; + } + } + } + if (ourItr.hasNext()) { + return 1; + } else if (theirItr.hasNext()) { + return -1; + } else { + return 0; + } + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java index 0370ae5..f6bf635 100644 --- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java +++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java @@ -44,12 +44,12 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.ShortWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; import org.apache.orc.Reader; import org.apache.orc.RecordReader; import org.apache.orc.TypeDescription; -public class OrcRecordReader<V extends Writable> +public class OrcRecordReader<V extends WritableComparable> implements org.apache.hadoop.mapred.RecordReader<NullWritable, V> { private final TypeDescription schema; private final RecordReader batchReader; @@ -501,10 +501,10 @@ public class OrcRecordReader<V extends Writable> } } - static Writable nextValue(ColumnVector vector, - int row, - TypeDescription schema, - Object previous) { + static WritableComparable nextValue(ColumnVector vector, + int row, + TypeDescription schema, + Object previous) { switch (schema.getCategory()) { case BOOLEAN: return nextBoolean(vector, row, previous); http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java index 74b3b28..2dd749b 100644 --- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java +++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java @@ -28,28 +28,28 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.ShortWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; import org.apache.orc.TypeDescription; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -public final class OrcStruct implements Writable { +public final class OrcStruct implements WritableComparable<OrcStruct> { - private Writable[] fields; + private WritableComparable[] fields; private final TypeDescription schema; public OrcStruct(TypeDescription schema) { this.schema = schema; - fields = new Writable[schema.getChildren().size()]; + fields = new WritableComparable[schema.getChildren().size()]; } - public Writable getFieldValue(int fieldIndex) { + public WritableComparable getFieldValue(int fieldIndex) { return fields[fieldIndex]; } - public void setFieldValue(int fieldIndex, Writable value) { + public void setFieldValue(int fieldIndex, WritableComparable value) { fields[fieldIndex] = value; } @@ -59,10 +59,10 @@ public final class OrcStruct implements Writable { @Override public void write(DataOutput output) throws IOException { - for(int f=0; f < fields.length; ++f) { - output.writeBoolean(fields[f] != null); - if (fields[f] != null) { - fields[f].write(output); + for(WritableComparable field: fields) { + output.writeBoolean(field != null); + if (field != null) { + field.write(output); } } } @@ -81,7 +81,21 @@ public final class OrcStruct implements Writable { } } - public void setFieldValue(String fieldName, Writable value) { + /** + * Set all of the fields in the struct + * @param values the list of values for each of the fields. + */ + public void setAllFields(WritableComparable... values) { + if (fields.length != values.length) { + throw new IllegalArgumentException("Wrong number (" + values.length + + ") of fields for " + schema); + } + for (int col = 0; col < fields.length && col < values.length; ++col) { + fields[col] = values[col]; + } + } + + public void setFieldValue(String fieldName, WritableComparable value) { int fieldIdx = schema.getFieldNames().indexOf(fieldName); if (fieldIdx == -1) { throw new IllegalArgumentException("Field " + fieldName + @@ -90,7 +104,7 @@ public final class OrcStruct implements Writable { fields[fieldIdx] = value; } - public Writable getFieldValue(String fieldName) { + public WritableComparable getFieldValue(String fieldName) { int fieldIdx = schema.getFieldNames().indexOf(fieldName); if (fieldIdx == -1) { throw new IllegalArgumentException("Field " + fieldName + @@ -150,7 +164,7 @@ public final class OrcStruct implements Writable { /* Routines for stubbing into Writables */ - public static Writable createValue(TypeDescription type) { + public static WritableComparable createValue(TypeDescription type) { switch (type.getCategory()) { case BOOLEAN: return new BooleanWritable(); case BYTE: return new ByteWritable(); @@ -185,4 +199,30 @@ public final class OrcStruct implements Writable { throw new IllegalArgumentException("Unknown type " + type); } } + + @Override + public int compareTo(OrcStruct other) { + if (other == null) { + return -1; + } + int result = schema.compareTo(other.schema); + if (result != 0) { + return result; + } + for(int c = 0; c < fields.length && c < other.fields.length; ++c) { + if (fields[c] == null) { + if (other.fields[c] != null) { + return 1; + } + } else if (other.fields[c] == null) { + return -1; + } else { + int val = fields[c].compareTo(other.fields[c]); + if (val != 0) { + return val; + } + } + } + return fields.length - other.fields.length; + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java index 200a966..ee97b9f 100644 --- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java +++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java @@ -17,17 +17,19 @@ */ package org.apache.orc.mapred; -import org.apache.hadoop.io.Writable; + +import org.apache.hadoop.io.WritableComparable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.sql.Timestamp; +import java.util.Date; /** * A Timestamp implementation that implements Writable. */ -public class OrcTimestamp extends Timestamp implements Writable { +public class OrcTimestamp extends Timestamp implements WritableComparable<Date> { public OrcTimestamp() { super(0); http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java index 3e7c909..6eb0e6a 100644 --- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java +++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java @@ -18,6 +18,7 @@ package org.apache.orc.mapred; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; import org.apache.orc.TypeDescription; import java.io.DataInput; @@ -27,17 +28,17 @@ import java.io.IOException; /** * An in-memory representation of a union type. */ -public final class OrcUnion implements Writable { +public final class OrcUnion implements WritableComparable<OrcUnion> { private byte tag; - private Writable object; + private WritableComparable object; private final TypeDescription schema; public OrcUnion(TypeDescription schema) { this.schema = schema; } - public void set(byte tag, Writable object) { - this.tag = tag; + public void set(int tag, WritableComparable object) { + this.tag = (byte) tag; this.object = object; } @@ -100,4 +101,22 @@ public final class OrcUnion implements Writable { object = null; } } + + @Override + public int compareTo(OrcUnion other) { + if (other == null) { + return -1; + } + int result = schema.compareTo(other.schema); + if (result != 0) { + return result; + } + if (tag != other.tag) { + return tag - other.tag; + } + if (object == null) { + return other.object == null ? 0 : 1; + } + return object.compareTo(other.object); + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java index 4ac7ca9..b0cdfb7 100644 --- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java +++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java @@ -21,6 +21,7 @@ package org.apache.orc.mapred; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.orc.TypeDescription; import org.junit.Test; @@ -62,4 +63,48 @@ public class TestOrcList { cloneWritable(expected, actual); assertEquals(expected, actual); } + + @Test + public void testCompare() { + TypeDescription schema = TypeDescription.fromString("array<string>"); + OrcList<Text> left = new OrcList<>(schema); + assertEquals(-1 ,left.compareTo(null)); + OrcList<Text> right = new OrcList<>(schema); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + right.add(new Text("aa")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + left.add(new Text("aa")); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + left.add(new Text("bb")); + right.add(new Text("cc")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + left.clear(); + right.clear(); + left.add(null); + right.add(null); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + right.clear(); + right.add(new Text("ddd")); + assertEquals(1, left.compareTo(right)); + assertEquals(-1, right.compareTo(left)); + } + + @Test + public void testSchemaInCompare() { + TypeDescription leftType = TypeDescription.fromString("array<int>"); + TypeDescription rightType = TypeDescription.fromString("array<string>"); + OrcList leftList = new OrcList(leftType); + OrcList rightList = new OrcList(rightType); + assertEquals(-4, leftList.compareTo(rightList)); + assertEquals(4, rightList.compareTo(leftList)); + leftList.add(new IntWritable(123)); + rightList.add(new Text("123")); + assertEquals(-4, leftList.compareTo(rightList)); + assertEquals(4, rightList.compareTo(leftList)); + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java index 34e1feb..d34b72e 100644 --- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java +++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java @@ -18,11 +18,9 @@ package org.apache.orc.mapred; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.Text; import org.apache.orc.TypeDescription; import org.junit.Test; @@ -54,4 +52,138 @@ public class TestOrcMap { TestOrcList.cloneWritable(expected, actual); assertEquals(expected, actual); } + + @Test + public void testCompare() { + TypeDescription schema = TypeDescription.fromString("map<string,string>"); + OrcMap<Text,Text> left = new OrcMap<>(schema); + assertEquals(-1 ,left.compareTo(null)); + OrcMap<Text,Text> right = new OrcMap<>(schema); + + // empty maps + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + + // {} vs {"aa" -> null} + right.put(new Text("aa"), null); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + + // {"aa" -> null} vs {"aa" -> null} + left.put(new Text("aa"), null); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + + // {"aa" -> "bb"} vs {"aa" -> "bb"} + left.put(new Text("aa"), new Text("bb")); + right.put(new Text("aa"), new Text("bb")); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + + // {"aa" -> "bb"} vs {"aa" -> "cc"} + right.put(new Text("aa"), new Text("cc")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + + // {"aa" -> "bb"} vs {"a" -> "zzz", "aa" -> "cc"} + right.put(new Text("a"), new Text("zzz")); + assertEquals(1, left.compareTo(right)); + assertEquals(-1, right.compareTo(left)); + + // {"aa" -> null} vs {"aa" -> "bb"} + left.put(new Text("aa"), null); + right.remove(new Text("a")); + right.put(new Text("aa"), new Text("cc")); + assertEquals(1, left.compareTo(right)); + assertEquals(-1, right.compareTo(left)); + + // {"aa" -> null, "bb" -> "cc"} vs {"aa" -> null, "bb" -> "dd"} + left.put(new Text("aa"), null); + left.put(new Text("bb"), new Text("cc")); + right.put(new Text("aa"), null); + right.put(new Text("bb"), new Text("dd")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + } + + @Test + public void testStructKeys() { + TypeDescription schema = TypeDescription.fromString("map<struct<i:int>,string>"); + OrcMap<OrcStruct, Text> map = new OrcMap<>(schema); + OrcStruct struct = new OrcStruct(schema.getChildren().get(0)); + struct.setFieldValue(0, new IntWritable(12)); + map.put(struct, new Text("a")); + assertEquals("a", map.get(struct).toString()); + struct = new OrcStruct(schema.getChildren().get(0)); + struct.setFieldValue(0, new IntWritable(14)); + map.put(struct, new Text("b")); + assertEquals(2, map.size()); + } + + @Test + public void testListKeys() { + TypeDescription schema = TypeDescription.fromString("map<array<int>,string>"); + OrcMap<OrcList, Text> map = new OrcMap<>(schema); + OrcList<IntWritable> list = new OrcList<>(schema.getChildren().get(0)); + list.add(new IntWritable(123)); + map.put(list, new Text("a")); + assertEquals("a", map.get(list).toString()); + list = new OrcList<>(schema.getChildren().get(0)); + list.add(new IntWritable(333)); + map.put(list, new Text("b")); + assertEquals(2, map.size()); + assertEquals("b", map.get(list).toString()); + } + + @Test + public void testUnionKeys() { + TypeDescription schema = TypeDescription.fromString("map<uniontype<int,string>,string>"); + OrcMap<OrcUnion, Text> map = new OrcMap<>(schema); + OrcUnion un = new OrcUnion(schema.getChildren().get(0)); + un.set(0, new IntWritable(123)); + map.put(un, new Text("hi")); + un = new OrcUnion(schema.getChildren().get(0)); + un.set(1, new Text("aaaa")); + map.put(un, new Text("google")); + assertEquals(2, map.size()); + assertEquals("google", map.get(un).toString()); + } + + @Test + public void testMapKeys() { + TypeDescription schema = TypeDescription.fromString("map<map<string,string>,string>"); + OrcMap<OrcMap<Text,Text>, Text> left = new OrcMap<>(schema); + + assertEquals(-1, left.compareTo(null)); + + OrcMap<OrcMap<Text,Text>, Text> right = new OrcMap<>(schema); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + + OrcMap<Text,Text> item = new OrcMap<>(schema.getChildren().get(0)); + item.put(new Text("aa"), new Text("bb")); + left.put(item, new Text("cc")); + assertEquals(1, left.compareTo(right)); + assertEquals(-1, right.compareTo(left)); + + item = new OrcMap<>(schema.getChildren().get(0)); + item.put(new Text("aa"), new Text("dd")); + right.put(item, new Text("bb")); + assertEquals(-2, left.compareTo(right)); + assertEquals(2, right.compareTo(left)); + } + + @Test + public void testSchemaInCompare() { + TypeDescription leftType = TypeDescription.fromString("map<string,int>"); + TypeDescription rightType = TypeDescription.fromString("map<string,string>"); + OrcMap left = new OrcMap(leftType); + OrcMap right = new OrcMap(rightType); + assertEquals(-4, left.compareTo(right)); + assertEquals(4, right.compareTo(left)); + left.put(new Text("123"), new IntWritable(123)); + right.put(new Text("123"), new Text("123")); + assertEquals(-4, left.compareTo(right)); + assertEquals(4, right.compareTo(left)); + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java index 4af6a1c..d32ce94 100644 --- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java +++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java @@ -93,4 +93,48 @@ public class TestOrcStruct { thrown.expect(IllegalArgumentException.class); struct.setFieldValue("bad", new Text("foobar")); } + + @Test + public void testCompare() { + OrcStruct left = new OrcStruct(TypeDescription.fromString + ("struct<i:int,j:string>")); + assertEquals(-1 ,left.compareTo(null)); + OrcStruct right = new OrcStruct(TypeDescription.fromString + ("struct<i:int,j:string,k:int>")); + left.setFieldValue(0, new IntWritable(10)); + right.setFieldValue(0, new IntWritable(12)); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + right.setFieldValue(0, new IntWritable(10)); + left.setFieldValue(1, new Text("a")); + right.setFieldValue(1, new Text("b")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + right.setFieldValue(1, new Text("a")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + right = new OrcStruct(TypeDescription.fromString + ("struct<i:int,j:string>")); + left.setFieldValue(0, null); + left.setFieldValue(1, null); + assertEquals(0, left.compareTo(right)); + assertEquals(0, right.compareTo(left)); + right.setFieldValue(0, new IntWritable(12)); + assertEquals(1 , left.compareTo(right)); + assertEquals(-1, right.compareTo(left)); + } + + @Test + public void testSchemaInCompare() { + TypeDescription leftType = TypeDescription.fromString("struct<s:string,i:int>"); + TypeDescription rightType = TypeDescription.fromString("struct<s:string,j:bigint>"); + OrcStruct left = new OrcStruct(leftType); + OrcStruct right = new OrcStruct(rightType); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + left.setAllFields(new Text("123"), new IntWritable(123)); + right.setAllFields(new Text("123"), new LongWritable(456)); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java index 82fd94f..7c8d523 100644 --- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java +++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java @@ -55,4 +55,37 @@ public class TestOrcUnion { TestOrcList.cloneWritable(expected, actual); assertEquals(expected, actual); } + + @Test + public void testCompare() { + TypeDescription schema = + TypeDescription.fromString("uniontype<int,string,bigint>"); + OrcUnion left = new OrcUnion(schema); + OrcUnion right = new OrcUnion(schema); + assertEquals(-1 ,left.compareTo(null)); + assertEquals(0, left.compareTo(right)); + + left.set(1, new IntWritable(10)); + right.set(1, new IntWritable(12)); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + + right.set(2, new Text("a")); + assertEquals(-1, left.compareTo(right)); + assertEquals(1, right.compareTo(left)); + } + + @Test + public void testSchemaInCompare() { + TypeDescription leftType = TypeDescription.fromString("uniontype<string,tinyint>"); + TypeDescription rightType = TypeDescription.fromString("uniontype<string,bigint>"); + OrcUnion left = new OrcUnion(leftType); + OrcUnion right = new OrcUnion(rightType); + assertEquals(-3, left.compareTo(right)); + assertEquals(3, right.compareTo(left)); + left.set(0, new Text("123")); + right.set(0, new Text("1")); + assertEquals(-3, left.compareTo(right)); + assertEquals(3, right.compareTo(left)); + } } http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/java/mapreduce/src/test/resources/log4j.properties b/java/mapreduce/src/test/resources/log4j.properties new file mode 100644 index 0000000..67e071d --- /dev/null +++ b/java/mapreduce/src/test/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO,stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%p\t%d{ISO8601}\t%r\t%c\t[%t]\t%m%n http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/pom.xml ---------------------------------------------------------------------- diff --git a/java/pom.xml b/java/pom.xml index 008c9bf..9941dee 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -92,6 +92,7 @@ <TZ>US/Pacific</TZ> <LANG>en_US.UTF-8</LANG> </environmentVariables> + <failIfNoTests>false</failIfNoTests> <systemPropertyVariables> <test.tmp.dir>${test.tmp.dir}</test.tmp.dir> </systemPropertyVariables> http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/storage-api/pom.xml ---------------------------------------------------------------------- diff --git a/java/storage-api/pom.xml b/java/storage-api/pom.xml index 46f534f..46e9c9e 100644 --- a/java/storage-api/pom.xml +++ b/java/storage-api/pom.xml @@ -73,6 +73,18 @@ <target>1.7</target> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <reuseForks>false</reuseForks> + <argLine>-Xmx2048m -XX:MaxPermSize=512m</argLine> + <failIfNoTests>false</failIfNoTests> + <systemPropertyVariables> + <test.tmp.dir>${test.tmp.dir}</test.tmp.dir> + </systemPropertyVariables> + </configuration> + </plugin> </plugins> </build>
