Repository: arrow Updated Branches: refs/heads/master c3cfa3d3b -> 7fb4d24a3
ARROW-315: finalize timestamp Author: Julien Le Dem <[email protected]> Closes #156 from julienledem/timestamp and squashes the following commits: 0ee017f [Julien Le Dem] review feedback 86cae98 [Julien Le Dem] ARROW-315: finalize timestamp Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/7fb4d24a Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/7fb4d24a Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/7fb4d24a Branch: refs/heads/master Commit: 7fb4d24a35269db99fa112c0512d4a32c372dd74 Parents: c3cfa3d Author: Julien Le Dem <[email protected]> Authored: Tue Oct 4 15:11:56 2016 -0700 Committer: Julien Le Dem <[email protected]> Committed: Tue Oct 4 15:11:56 2016 -0700 ---------------------------------------------------------------------- format/Message.fbs | 5 ++- .../vector/src/main/codegen/data/ArrowTypes.tdd | 2 +- .../codegen/templates/NullableValueVectors.java | 2 +- .../org/apache/arrow/vector/types/Types.java | 46 +++++++++++--------- .../apache/arrow/vector/pojo/TestConvert.java | 3 +- 5 files changed, 34 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/7fb4d24a/format/Message.fbs ---------------------------------------------------------------------- diff --git a/format/Message.fbs b/format/Message.fbs index 3d877a2..d8fa650 100644 --- a/format/Message.fbs +++ b/format/Message.fbs @@ -64,8 +64,11 @@ table Date { table Time { } +enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND } + +/// time from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC. table Timestamp { - timezone: string; + unit: TimeUnit; } enum IntervalUnit: short { YEAR_MONTH, DAY_TIME} http://git-wip-us.apache.org/repos/asf/arrow/blob/7fb4d24a/java/vector/src/main/codegen/data/ArrowTypes.tdd ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd index 9624fec..11ac99a 100644 --- a/java/vector/src/main/codegen/data/ArrowTypes.tdd +++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd @@ -66,7 +66,7 @@ }, { name: "Timestamp", - fields: [{name: "timezone", type: "String"}] + fields: [{name: "unit", type: short}] }, { name: "Interval", http://git-wip-us.apache.org/repos/asf/arrow/blob/7fb4d24a/java/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 8f325af..bafa317 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -103,7 +103,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#elseif minor.class == "Float8"> field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null); <#elseif minor.class == "TimeStamp"> - field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null); + field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null); <#elseif minor.class == "IntervalDay"> field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null); <#elseif minor.class == "IntervalYear"> http://git-wip-us.apache.org/repos/asf/arrow/blob/7fb4d24a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 2ff93d4..d959367 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -19,6 +19,7 @@ package org.apache.arrow.vector.types; import org.apache.arrow.flatbuf.IntervalUnit; import org.apache.arrow.flatbuf.Precision; +import org.apache.arrow.flatbuf.TimeUnit; import org.apache.arrow.flatbuf.UnionMode; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; @@ -101,7 +102,7 @@ public class Types { private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null); private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null); private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null); - private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null); + private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null); private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null); private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null); private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null); @@ -143,8 +144,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new NullableMapWriter((NullableMapVector) vector); } - }, // an empty map column. Useful for conceptual setup. Children listed within here - + }, TINYINT(new Int(8, true)) { @Override public Field getField() { @@ -160,7 +160,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new TinyIntWriterImpl((NullableTinyIntVector) vector); } - }, // single byte signed integer + }, SMALLINT(new Int(16, true)) { @Override public Field getField() { @@ -176,7 +176,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new SmallIntWriterImpl((NullableSmallIntVector) vector); } - }, // two byte signed integer + }, INT(new Int(32, true)) { @Override public Field getField() { @@ -192,7 +192,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new IntWriterImpl((NullableIntVector) vector); } - }, // four byte signed integer + }, BIGINT(new Int(64, true)) { @Override public Field getField() { @@ -208,7 +208,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new BigIntWriterImpl((NullableBigIntVector) vector); } - }, // eight byte signed integer + }, DATE(Date.INSTANCE) { @Override public Field getField() { @@ -224,7 +224,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new DateWriterImpl((NullableDateVector) vector); } - }, // days since 4713bc + }, TIME(Time.INSTANCE) { @Override public Field getField() { @@ -240,8 +240,9 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new TimeWriterImpl((NullableTimeVector) vector); } - }, // time in micros before or after 2000/1/1 - TIMESTAMP(new Timestamp("")) { + }, + // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC. + TIMESTAMP(new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND)) { @Override public Field getField() { return TIMESTAMP_FIELD; @@ -289,6 +290,7 @@ public class Types { return new IntervalYearWriterImpl((NullableIntervalYearVector) vector); } }, + // 4 byte ieee 754 FLOAT4(new FloatingPoint(Precision.SINGLE)) { @Override public Field getField() { @@ -304,7 +306,8 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new Float4WriterImpl((NullableFloat4Vector) vector); } - }, // 4 byte ieee 754 + }, + // 8 byte ieee 754 FLOAT8(new FloatingPoint(Precision.DOUBLE)) { @Override public Field getField() { @@ -320,7 +323,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new Float8WriterImpl((NullableFloat8Vector) vector); } - }, // 8 byte ieee 754 + }, BIT(Bool.INSTANCE) { @Override public Field getField() { @@ -336,7 +339,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new BitWriterImpl((NullableBitVector) vector); } - }, // single bit value (boolean) + }, VARCHAR(Utf8.INSTANCE) { @Override public Field getField() { @@ -352,7 +355,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new VarCharWriterImpl((NullableVarCharVector) vector); } - }, // utf8 variable length string + }, VARBINARY(Binary.INSTANCE) { @Override public Field getField() { @@ -368,7 +371,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new VarBinaryWriterImpl((NullableVarBinaryVector) vector); } - }, // variable length binary + }, DECIMAL(null) { @Override public ArrowType getType() { @@ -388,7 +391,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new DecimalWriterImpl((NullableDecimalVector) vector); } - }, // variable length binary + }, UINT1(new Int(8, false)) { @Override public Field getField() { @@ -404,7 +407,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new UInt1WriterImpl((NullableUInt1Vector) vector); } - }, // unsigned 1 byte integer + }, UINT2(new Int(16, false)) { @Override public Field getField() { @@ -420,7 +423,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new UInt2WriterImpl((NullableUInt2Vector) vector); } - }, // unsigned 2 byte integer + }, UINT4(new Int(32, false)) { @Override public Field getField() { @@ -436,7 +439,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new UInt4WriterImpl((NullableUInt4Vector) vector); } - }, // unsigned 4 byte integer + }, UINT8(new Int(64, false)) { @Override public Field getField() { @@ -452,7 +455,7 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new UInt8WriterImpl((NullableUInt8Vector) vector); } - }, // unsigned 8 byte integer + }, LIST(List.INSTANCE) { @Override public Field getField() { @@ -576,6 +579,9 @@ public class Types { } @Override public MinorType visit(Timestamp type) { + if (type.getUnit() != TimeUnit.MILLISECOND) { + throw new UnsupportedOperationException("Only milliseconds supported: " + type); + } return MinorType.TIMESTAMP; } http://git-wip-us.apache.org/repos/asf/arrow/blob/7fb4d24a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index ed740cd..3da8db2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -21,6 +21,7 @@ import static org.apache.arrow.flatbuf.Precision.DOUBLE; import static org.apache.arrow.flatbuf.Precision.SINGLE; import static org.junit.Assert.assertEquals; +import org.apache.arrow.flatbuf.TimeUnit; import org.apache.arrow.flatbuf.UnionMode; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint; @@ -80,7 +81,7 @@ public class TestConvert { new Field("child4.1", true, Utf8.INSTANCE, null) ))); childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of( - new Field("child5.1", true, new Timestamp("UTC"), null), + new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null), new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of()) ))); Schema initialSchema = new Schema(childrenBuilder.build());
