Repository: arrow Updated Branches: refs/heads/master 720d422fa -> 08f38d979
ARROW-477: [Java] Add support for second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer Changes include: - add support for TimeStamp data type with second/microsecond/nanosecond time units - add an additional readLong() method to timestamp readers to support reading raw long values - add a simple test case for timestamp readers and writers Author: Jingyuan Wang <[email protected]> Closes #303 from alphalfalfa/arrow-477 and squashes the following commits: 0199574 [Jingyuan Wang] rename TimeStamp to TimeStampMilli 068e47f [Jingyuan Wang] use a test value that exhibits micro/nanosecond truncation when converting timestamps to JODA DateTime bef2330 [Jingyuan Wang] fix a typo 9b4d7b4 [Jingyuan Wang] add support for timestamp data type with second/microsecond/nanosecond time units Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/08f38d97 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/08f38d97 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/08f38d97 Branch: refs/heads/master Commit: 08f38d97904e8d265dea09cdc67946119998e039 Parents: 720d422 Author: Jingyuan Wang <[email protected]> Authored: Fri Feb 3 23:00:25 2017 -0500 Committer: Wes McKinney <[email protected]> Committed: Fri Feb 3 23:00:25 2017 -0500 ---------------------------------------------------------------------- .../src/main/codegen/data/ValueVectorTypes.tdd | 5 +- .../main/codegen/templates/ComplexReaders.java | 10 +++ .../codegen/templates/FixedValueVectors.java | 31 ++++++- .../codegen/templates/NullableValueVectors.java | 8 +- .../arrow/vector/file/json/JsonFileReader.java | 18 +++- .../arrow/vector/file/json/JsonFileWriter.java | 18 +++- .../org/apache/arrow/vector/types/Types.java | 88 +++++++++++++++++--- .../complex/writer/TestComplexWriter.java | 85 +++++++++++++++++++ .../apache/arrow/vector/file/BaseFileTest.java | 10 +-- .../apache/arrow/vector/pojo/TestConvert.java | 2 +- 10 files changed, 250 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/data/ValueVectorTypes.tdd ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index f7790bb..2181cfd 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -71,7 +71,10 @@ { class: "UInt8" }, { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], }, { class: "Date", javaType: "long", friendlyType: "DateTime" }, - { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" } + { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" } + { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" } + { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" } + { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" } ] }, { http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/templates/ComplexReaders.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 74a19a6..d537445 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -96,6 +96,16 @@ public class ${name}ReaderImpl extends AbstractFieldReader { public ${friendlyType} read${safeType}(){ return vector.getAccessor().getObject(idx()); } + + <#if minor.class == "TimeStampSec" || + minor.class == "TimeStampMilli" || + minor.class == "TimeStampMicro" || + minor.class == "TimeStampNano"> + @Override + public ${minor.boxedType} read${minor.boxedType}(){ + return vector.getAccessor().get(idx()); + } + </#if> public void copyValue(FieldWriter w){ http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java index be385d1..d5265f1 100644 --- a/java/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java @@ -490,7 +490,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return date; } - <#elseif minor.class == "TimeStamp"> + <#elseif minor.class == "TimeStampSec"> + @Override + public ${friendlyType} getObject(int index) { + long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index)); + org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC); + date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + return date; + } + + <#elseif minor.class == "TimeStampMilli"> @Override public ${friendlyType} getObject(int index) { org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); @@ -498,6 +507,26 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return date; } + <#elseif minor.class == "TimeStampMicro"> + @Override + public ${friendlyType} getObject(int index) { + // value is truncated when converting microseconds to milliseconds in order to use DateTime type + long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index)); + org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC); + date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + return date; + } + + <#elseif minor.class == "TimeStampNano"> + @Override + public ${friendlyType} getObject(int index) { + // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type + long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index)); + org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC); + date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + return date; + } + <#elseif minor.class == "IntervalYear"> @Override public ${friendlyType} getObject(int index) { http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 6a9ce65..ce63710 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -102,8 +102,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null); <#elseif minor.class == "Float8"> field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null); - <#elseif minor.class == "TimeStamp"> + <#elseif minor.class == "TimeStampSec"> + field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND), null); + <#elseif minor.class == "TimeStampMilli"> field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null); + <#elseif minor.class == "TimeStampMicro"> + field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND), null); + <#elseif minor.class == "TimeStampNano"> + field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND), null); <#elseif minor.class == "IntervalDay"> field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null); <#elseif minor.class == "IntervalYear"> http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 152867c..71fe88e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -37,7 +37,10 @@ import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.UInt1Vector; import org.apache.arrow.vector.UInt2Vector; @@ -199,9 +202,18 @@ public class JsonFileReader implements AutoCloseable { case VARCHAR: ((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8)); break; - case TIMESTAMP: - ((TimeStampVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + case TIMESTAMPSEC: + ((TimeStampSecVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); break; + case TIMESTAMPMILLI: + ((TimeStampMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICRO: + ((TimeStampMicroVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANO: + ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; default: throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType()); } http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 6ff3577..ddc8043 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -24,7 +24,10 @@ import java.util.List; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.BufferBacked; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ValueVector.Accessor; import org.apache.arrow.vector.VectorSchemaRoot; @@ -139,8 +142,17 @@ public class JsonFileWriter implements AutoCloseable { private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException { switch (valueVector.getMinorType()) { - case TIMESTAMP: - generator.writeNumber(((TimeStampVector)valueVector).getAccessor().get(i)); + case TIMESTAMPSEC: + generator.writeNumber(((TimeStampSecVector)valueVector).getAccessor().get(i)); + break; + case TIMESTAMPMILLI: + generator.writeNumber(((TimeStampMilliVector)valueVector).getAccessor().get(i)); + break; + case TIMESTAMPMICRO: + generator.writeNumber(((TimeStampMicroVector)valueVector).getAccessor().get(i)); + break; + case TIMESTAMPNANO: + generator.writeNumber(((TimeStampNanoVector)valueVector).getAccessor().get(i)); break; case BIT: generator.writeNumber(((BitVector)valueVector).getAccessor().get(i)); http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 2a2fb74..ab539d5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -33,7 +33,10 @@ import org.apache.arrow.vector.NullableIntVector; import org.apache.arrow.vector.NullableIntervalDayVector; import org.apache.arrow.vector.NullableIntervalYearVector; import org.apache.arrow.vector.NullableSmallIntVector; -import org.apache.arrow.vector.NullableTimeStampVector; +import org.apache.arrow.vector.NullableTimeStampSecVector; +import org.apache.arrow.vector.NullableTimeStampMilliVector; +import org.apache.arrow.vector.NullableTimeStampMicroVector; +import org.apache.arrow.vector.NullableTimeStampNanoVector; import org.apache.arrow.vector.NullableTimeVector; import org.apache.arrow.vector.NullableTinyIntVector; import org.apache.arrow.vector.NullableUInt1Vector; @@ -58,7 +61,10 @@ import org.apache.arrow.vector.complex.impl.IntervalDayWriterImpl; import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl; import org.apache.arrow.vector.complex.impl.NullableMapWriter; import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl; -import org.apache.arrow.vector.complex.impl.TimeStampWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl; import org.apache.arrow.vector.complex.impl.TimeWriterImpl; import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl; import org.apache.arrow.vector.complex.impl.UInt1WriterImpl; @@ -102,7 +108,10 @@ public class Types { private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null); private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null); private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null); - private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null); + private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND), null); + private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null); + private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND), null); + private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND), null); private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null); private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null); private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null); @@ -241,21 +250,72 @@ public class Types { return new TimeWriterImpl((NullableTimeVector) vector); } }, + // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. + TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND)) { + @Override + public Field getField() { + return TIMESTAMPSEC_FIELD; + } + + @Override + public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) { + return new NullableTimeStampSecVector(name, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampSecWriterImpl((NullableTimeStampSecVector) vector); + } + }, // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC. - TIMESTAMP(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) { + TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) { @Override public Field getField() { - return TIMESTAMP_FIELD; + return TIMESTAMPMILLI_FIELD; } @Override public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) { - return new NullableTimeStampVector(name, allocator); + return new NullableTimeStampMilliVector(name, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampWriterImpl((NullableTimeStampVector) vector); + return new TimeStampMilliWriterImpl((NullableTimeStampMilliVector) vector); + } + }, + // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. + TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND)) { + @Override + public Field getField() { + return TIMESTAMPMICRO_FIELD; + } + + @Override + public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) { + return new NullableTimeStampMicroVector(name, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampMicroWriterImpl((NullableTimeStampMicroVector) vector); + } + }, + // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC. + TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND)) { + @Override + public Field getField() { + return TIMESTAMPNANO_FIELD; + } + + @Override + public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) { + return new NullableTimeStampNanoVector(name, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampNanoWriterImpl((NullableTimeStampNanoVector) vector); } }, INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) { @@ -579,10 +639,18 @@ public class Types { } @Override public MinorType visit(Timestamp type) { - if (type.getUnit() != TimeUnit.MILLISECOND) { - throw new UnsupportedOperationException("Only milliseconds supported: " + type); + switch (type.getUnit()) { + case SECOND: + return MinorType.TIMESTAMPSEC; + case MILLISECOND: + return MinorType.TIMESTAMPMILLI; + case MICROSECOND: + return MinorType.TIMESTAMPMICRO; + case NANOSECOND: + return MinorType.TIMESTAMPNANO; + default: + throw new IllegalArgumentException("unknown unit: " + type); } - return MinorType.TIMESTAMP; } @Override http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 2c0c853..7a2d416 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -43,12 +43,15 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.ArrowType.Int; import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.Text; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.junit.Assert; import org.junit.Test; @@ -561,4 +564,86 @@ public class TestComplexWriter { Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); } + + @Test + public void timeStampWriters() throws Exception { + // test values + final long expectedNanos = 981173106123456789L; + final long expectedMicros = 981173106123456L; + final long expectedMillis = 981173106123L; + final long expectedSecs = 981173106L; + final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault()); + final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault()); + final DateTime expectedMicroDateTime = expectedMilliDateTime; + final DateTime expectedNanoDateTime = expectedMilliDateTime; + + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + + TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec"); + timeStampSecWriter.setPosition(0); + timeStampSecWriter.writeTimeStampSec(expectedSecs); + + TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli"); + timeStampWriter.setPosition(1); + timeStampWriter.writeTimeStampMilli(expectedMillis); + + TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro"); + timeStampMicroWriter.setPosition(2); + timeStampMicroWriter.writeTimeStampMicro(expectedMicros); + + TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano"); + timeStampNanoWriter.setPosition(3); + timeStampNanoWriter.writeTimeStampNano(expectedNanos); + + // schema + Field secField = parent.getField().getChildren().get(0).getChildren().get(0); + Assert.assertEquals("sec", secField.getName()); + Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, secField.getType().getTypeID()); + + Field milliField = parent.getField().getChildren().get(0).getChildren().get(1); + Assert.assertEquals("milli", milliField.getName()); + Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, milliField.getType().getTypeID()); + + Field microField = parent.getField().getChildren().get(0).getChildren().get(2); + Assert.assertEquals("micro", microField.getName()); + Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, microField.getType().getTypeID()); + + Field nanoField = parent.getField().getChildren().get(0).getChildren().get(3); + Assert.assertEquals("nano", nanoField.getName()); + Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, nanoField.getType().getTypeID()); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + + FieldReader secReader = rootReader.reader("sec"); + secReader.setPosition(0); + DateTime secDateTime = secReader.readDateTime(); + Assert.assertEquals(expectedSecDateTime, secDateTime); + long secLong = secReader.readLong(); + Assert.assertEquals(expectedSecs, secLong); + + FieldReader milliReader = rootReader.reader("milli"); + milliReader.setPosition(1); + DateTime milliDateTime = milliReader.readDateTime(); + Assert.assertEquals(expectedMilliDateTime, milliDateTime); + long milliLong = milliReader.readLong(); + Assert.assertEquals(expectedMillis, milliLong); + + FieldReader microReader = rootReader.reader("micro"); + microReader.setPosition(2); + DateTime microDateTime = microReader.readDateTime(); + Assert.assertEquals(expectedMicroDateTime, microDateTime); + long microLong = microReader.readLong(); + Assert.assertEquals(expectedMicros, microLong); + + FieldReader nanoReader = rootReader.reader("nano"); + nanoReader.setPosition(3); + DateTime nanoDateTime = nanoReader.readDateTime(); + Assert.assertEquals(expectedNanoDateTime, nanoDateTime); + long nanoLong = nanoReader.readLong(); + Assert.assertEquals(expectedNanos, nanoLong); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 6e577b5..774bead 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -33,7 +33,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BigIntWriter; import org.apache.arrow.vector.complex.writer.IntWriter; -import org.apache.arrow.vector.holders.NullableTimeStampHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; import org.joda.time.DateTimeZone; import org.junit.After; import org.junit.Assert; @@ -100,7 +100,7 @@ public class BaseFileTest { listWriter.endList(); mapWriter.setPosition(i); mapWriter.start(); - mapWriter.timeStamp("timestamp").writeTimeStamp(i); + mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i); mapWriter.end(); } writer.setValueCount(count); @@ -130,7 +130,7 @@ public class BaseFileTest { } Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size()); - NullableTimeStampHolder h = new NullableTimeStampHolder(); + NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); FieldReader mapReader = root.getVector("map").getReader(); mapReader.setPosition(i); mapReader.reader("timestamp").read(h); @@ -167,7 +167,7 @@ public class BaseFileTest { Assert.assertEquals(i % 3, unionReader.size()); break; case 3: - NullableTimeStampHolder h = new NullableTimeStampHolder(); + NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); unionReader.reader("timestamp").read(h); Assert.assertEquals(i, h.value); break; @@ -209,7 +209,7 @@ public class BaseFileTest { case 3: mapWriter.setPosition(i); mapWriter.start(); - mapWriter.timeStamp("timestamp").writeTimeStamp(i); + mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i); mapWriter.end(); break; } http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 5a238bc..65823e2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -80,7 +80,7 @@ public class TestConvert { childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of( new Field("child4.1", true, Utf8.INSTANCE, null) ))); - childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of( + childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of( new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null), new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of()) )));
