Repository: arrow Updated Branches: refs/heads/master 4381845b5 -> 681afabb4
ARROW-977: [java] Add Timezone aware timestamp vectors Author: Julien Le Dem <jul...@apache.org> Closes #644 from julienledem/TZ and squashes the following commits: 37987b9 [Julien Le Dem] add integration tests a58fcae [Julien Le Dem] add integration test 39966aa [Julien Le Dem] add other vectors and tests bf245ce [Julien Le Dem] add TZ vectors Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/681afabb Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/681afabb Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/681afabb Branch: refs/heads/master Commit: 681afabb4fa138e7dab694fa63f60220810eb17e Parents: 4381845 Author: Julien Le Dem <jul...@apache.org> Authored: Mon May 15 16:16:53 2017 -0400 Committer: Wes McKinney <wes.mckin...@twosigma.com> Committed: Mon May 15 16:16:53 2017 -0400 ---------------------------------------------------------------------- integration/integration_test.py | 6 +- .../src/main/codegen/data/ValueVectorTypes.tdd | 36 ++- .../codegen/templates/AbstractFieldWriter.java | 10 +- .../AbstractPromotableFieldWriter.java | 31 ++- .../src/main/codegen/templates/BaseWriter.java | 4 +- .../main/codegen/templates/ComplexCopier.java | 7 +- .../codegen/templates/FixedValueVectors.java | 50 ++-- .../src/main/codegen/templates/MapWriters.java | 38 ++- .../codegen/templates/NullableValueVectors.java | 36 ++- .../main/codegen/templates/UnionListWriter.java | 2 +- .../src/main/codegen/templates/UnionWriter.java | 6 +- .../templates/VariableLengthVectors.java | 47 ++-- .../arrow/vector/file/json/JsonFileReader.java | 20 +- .../org/apache/arrow/vector/types/Types.java | 79 +++++-- .../complex/writer/TestComplexWriter.java | 232 ++++++++++++++----- .../apache/arrow/vector/file/BaseFileTest.java | 13 +- .../apache/arrow/vector/pojo/TestConvert.java | 3 +- 17 files changed, 429 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/integration/integration_test.py ---------------------------------------------------------------------- diff --git a/integration/integration_test.py b/integration/integration_test.py index 6466469..cc59593 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -620,7 +620,11 @@ def generate_datetime_case(): TimestampType('f7', 'ms'), TimestampType('f8', 'us'), TimestampType('f9', 'ns'), - TimestampType('f10', 'ms', tz=None) + TimestampType('f10', 'ms', tz=None), + TimestampType('f11', 's', tz='UTC'), + TimestampType('f12', 'ms', tz='US/Eastern'), + TimestampType('f13', 'us', tz='Europe/Paris'), + TimestampType('f14', 'ns', tz='US/Pacific') ] batch_sizes = [7, 10] http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/data/ValueVectorTypes.tdd ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index ca6d9ec..4d8c99e 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -71,12 +71,28 @@ minor: [ { class: "BigInt"}, { class: "UInt8" }, - { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], }, - { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, - { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, - { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, - { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, - { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}] }, + { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, + { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampSecTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.SECOND", "timezone"] }, + { class: "TimeStampMilliTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.MILLISECOND", "timezone"] }, + { class: "TimeStampMicroTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.MICROSECOND", "timezone"] }, + { class: "TimeStampNanoTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.NANOSECOND", "timezone"] }, { class: "TimeMicro" }, { class: "TimeNano" } ] @@ -97,7 +113,13 @@ boxedType: "ArrowBuf", minor: [ - { class: "Decimal", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] } + { + class: "Decimal", + maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", + typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal", + fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] + } ] }, { http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/AbstractFieldWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java index de076fc..65bcc05 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -59,7 +59,7 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr } <#if minor.class == "Decimal"> - public void writeDecimal(int start, ArrowBuf buffer) { + public void write${minor.class}(int start, ArrowBuf buffer) { fail("${name}"); } <#else> @@ -114,9 +114,11 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if minor.class?starts_with("Decimal") > - public ${capName}Writer ${lowerName}(String name, int scale, int precision) { - fail("${capName}"); + <#if minor.typeParams?? > + + @Override + public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { + fail("${capName}(" + <#list minor.typeParams as typeParam>"${typeParam.name}: " + ${typeParam.name} + ", " + </#list>")"); return null; } </#if> http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index ada0b1d..636b305 100644 --- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -72,31 +72,23 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter { } <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#if !minor.class?starts_with("Decimal") > + <#assign fields = minor.fields!type.fields /> @Override public void write(${name}Holder holder) { getWriter(MinorType.${name?upper_case}).write(holder); } + <#if minor.class == "Decimal"> + public void write${minor.class}(int start, ArrowBuf buffer) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(start, buffer); + } + <#else> public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); } - - <#else> - @Override - public void write(DecimalHolder holder) { - getWriter(MinorType.DECIMAL).write(holder); - } - - public void writeDecimal(int start, ArrowBuf buffer) { - getWriter(MinorType.DECIMAL).writeDecimal(start, buffer); - } - - </#if> + </#if> </#list></#list> - public void writeNull() { } @@ -125,10 +117,13 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter { <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if minor.class?starts_with("Decimal") > - public ${capName}Writer ${lowerName}(String name, int scale, int precision) { - return getWriter(MinorType.MAP).${lowerName}(name, scale, precision); + + <#if minor.typeParams?? > + @Override + public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { + return getWriter(MinorType.MAP).${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>); } + </#if> @Override public ${capName}Writer ${lowerName}(String name) { http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/BaseWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java index 3da02b0..405f466 100644 --- a/java/vector/src/main/codegen/templates/BaseWriter.java +++ b/java/vector/src/main/codegen/templates/BaseWriter.java @@ -53,8 +53,8 @@ public interface BaseWriter extends AutoCloseable, Positionable { <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if minor.class?starts_with("Decimal") > - ${capName}Writer ${lowerName}(String name, int scale, int precision); + <#if minor.typeParams?? > + ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>); </#if> ${capName}Writer ${lowerName}(String name); </#list></#list> http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/ComplexCopier.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java index fb7ae0f..518ad5d 100644 --- a/java/vector/src/main/codegen/templates/ComplexCopier.java +++ b/java/vector/src/main/codegen/templates/ComplexCopier.java @@ -72,7 +72,8 @@ public class ComplexCopier { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + + <#if !minor.typeParams?? > case ${name?upper_case}: if (reader.isSet()) { @@ -94,7 +95,7 @@ public class ComplexCopier { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name); </#if> @@ -113,7 +114,7 @@ public class ComplexCopier { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(); </#if> http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java index f403ecf..5d80b66 100644 --- a/java/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java @@ -25,9 +25,10 @@ import java.util.concurrent.TimeUnit; <#list vv.types as type> <#list type.minor as minor> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> +<#assign className = "${minor.class}Vector" /> <#if type.major == "Fixed"> -<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" /> +<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> <#include "/@includes/license.ftl" /> package org.apache.arrow.vector; @@ -43,8 +44,8 @@ package org.apache.arrow.vector; * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); +public final class ${className} extends BaseDataValueVector implements FixedWidthVector{ + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); public static final int TYPE_WIDTH = ${type.width}; @@ -53,24 +54,25 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width}; private int allocationMonitor = 0; + <#if minor.typeParams??> - <#if minor.class == "Decimal"> + <#list minor.typeParams as typeParam> + private final ${typeParam.type} ${typeParam.name}; + </#list> - private int precision; - private int scale; - - public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) { + public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { super(name, allocator); - this.precision = precision; - this.scale = scale; + <#list minor.typeParams as typeParam> + this.${typeParam.name} = ${typeParam.name}; + </#list> } <#else> - public ${minor.class}Vector(String name, BufferAllocator allocator) { + + public ${className}(String name, BufferAllocator allocator) { super(name, allocator); } </#if> - @Override public MinorType getMinorType() { return MinorType.${minor.class?upper_case}; @@ -219,17 +221,17 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${minor.class}Vector) to); + return new TransferImpl((${className}) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${className} target){ target.clear(); target.data = data.transferOwnership(target.allocator).buffer; target.data.writerIndex(data.writerIndex()); clear(); } - public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { + public void splitAndTransferTo(int startIndex, int length, ${className} target) { final int startPoint = startIndex * ${type.width}; final int sliceLength = length * ${type.width}; target.clear(); @@ -238,22 +240,18 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } private class TransferImpl implements TransferPair{ - private ${minor.class}Vector to; + private ${className} to; public TransferImpl(String name, BufferAllocator allocator){ - <#if minor.class == "Decimal"> - to = new ${minor.class}Vector(name, allocator, precision, scale); - <#else> - to = new ${minor.class}Vector(name, allocator); - </#if> + to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>, ${className}.this.${typeParam.name}</#list></#if>); } - public TransferImpl(${minor.class}Vector to) { + public TransferImpl(${className} to) { this.to = to; } @Override - public ${minor.class}Vector getTo(){ + public ${className} getTo(){ return to; } @@ -269,11 +267,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this); + to.copyFromSafe(fromIndex, toIndex, ${className}.this); } } - public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFrom(int fromIndex, int thisIndex, ${className} from){ <#if (type.width > 8 || minor.class == "IntervalDay")> from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width}); <#else> <#-- type.width <= 8 --> @@ -283,7 +281,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F </#if> <#-- type.width --> } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ while(thisIndex >= getValueCapacity()) { reAlloc(); } http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/MapWriters.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index d3e6de9..05048c5 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -56,7 +56,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter { </#if> this.container = container; for (Field child : container.getField().getChildren()) { - switch (Types.getMinorTypeForArrowType(child.getType())) { + MinorType minorType = Types.getMinorTypeForArrowType(child.getType()); + switch (minorType) { case MAP: map(child.getName()); break; @@ -71,15 +72,18 @@ public class ${mode}MapWriter extends AbstractFieldWriter { <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> - case ${upperName}: - <#if lowerName == "decimal" > - Decimal decimal = (Decimal)child.getType(); - decimal(child.getName(), decimal.getScale(), decimal.getPrecision()); + case ${upperName}: { + <#if minor.typeParams?? > + ${minor.arrowType} arrowType = (${minor.arrowType})child.getType(); + ${lowerName}(child.getName()<#list minor.typeParams as typeParam>, arrowType.get${typeParam.name?cap_first}()</#list>); <#else> ${lowerName}(child.getName()); - </#if> + </#if> break; + } </#list></#list> + default: + throw new UnsupportedOperationException("Unknown type: " + minorType); } } } @@ -205,7 +209,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter { <#assign vectName = capName /> <#assign vectName = "Nullable${capName}" /> - <#if minor.class?starts_with("Decimal") > + <#if minor.typeParams?? > + @Override public ${minor.class}Writer ${lowerName}(String name) { // returns existing writer final FieldWriter writer = fields.get(handleCase(name)); @@ -213,7 +218,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter { return writer; } - public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) { + @Override + public ${minor.class}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { <#else> @Override public ${minor.class}Writer ${lowerName}(String name) { @@ -223,7 +229,21 @@ public class ${mode}MapWriter extends AbstractFieldWriter { ValueVector vector; ValueVector currentVector = container.getChild(name); ${vectName}Vector v = container.addOrGet(name, - FieldType.nullable(<#if minor.class == "Decimal">new Decimal(precision, scale)<#else>MinorType.${upperName}.getType()</#if>), + FieldType.nullable( + <#if minor.typeParams??> + <#if minor.arrowTypeConstructorParams??> + <#assign constructorParams = minor.arrowTypeConstructorParams /> + <#else> + <#assign constructorParams = [] /> + <#list minor.typeParams as typeParam> + <#assign constructorParams = constructorParams + [ typeParam.name ] /> + </#list> + </#if> + new ${minor.arrowType}(${constructorParams?join(", ")}) + <#else> + MinorType.${upperName}.getType() + </#if> + ), ${vectName}Vector.class); writer = new PromotableWriter(v, container, getNullableMapWriterFactory()); vector = v; http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index ed2418e..3231c4c 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -61,12 +61,27 @@ public final class ${className} extends BaseDataValueVector implements <#if type private final List<BufferBacked> innerVectors; - <#if minor.class == "Decimal"> - private final int precision; - private final int scale; + <#if minor.typeParams??> + <#list minor.typeParams as typeParam> + private final ${typeParam.type} ${typeParam.name}; + </#list> - public ${className}(String name, BufferAllocator allocator, int precision, int scale) { - this(name, FieldType.nullable(new Decimal(precision, scale)), allocator); + /** + * Assumes the type is nullable and not dictionary encoded + * @param name name of the field + * @param allocator allocator to use to resize the vector<#list minor.typeParams as typeParam> + * @param ${typeParam.name} type parameter ${typeParam.name}</#list> + */ + public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { + <#if minor.arrowTypeConstructorParams??> + <#assign constructorParams = minor.arrowTypeConstructorParams /> + <#else> + <#assign constructorParams = [] /> + <#list minor.typeParams as typeParam> + <#assign constructorParams = constructorParams + [ typeParam.name ] /> + </#list> + </#if> + this(name, FieldType.nullable(new ${minor.arrowType}(${constructorParams?join(", ")})), allocator); } <#else> public ${className}(String name, BufferAllocator allocator) { @@ -76,11 +91,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type public ${className}(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator); - <#if minor.class == "Decimal"> - Decimal decimal = (Decimal)fieldType.getType(); - this.precision = decimal.getPrecision(); - this.scale = decimal.getScale(); - this.values = new ${valuesName}(valuesField, allocator, precision, scale); + <#if minor.typeParams??> + ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType(); + <#list minor.typeParams as typeParam> + this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}(); + </#list> + this.values = new ${valuesName}(valuesField, allocator<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>); <#else> this.values = new ${valuesName}(valuesField, allocator); </#if> http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/UnionListWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index d980830..d019a1e 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -94,7 +94,7 @@ public class UnionListWriter extends AbstractFieldWriter { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > @Override public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() { http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/UnionWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index 880f537..4a7c472 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -121,7 +121,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams??> case ${name?upper_case}: return get${name}Writer(); </#if> @@ -136,7 +136,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > private ${name}Writer ${name?uncap_first}Writer; @@ -206,7 +206,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > @Override public ${capName}Writer ${lowerName}(String name) { data.getMutator().setType(idx(), MinorType.MAP); http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java index 3d933ad..f13291b 100644 --- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.vector.VariableWidthVector; <#list type.minor as minor> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> +<#assign className = "${minor.class}Vector" /> <#if type.major == "VarLen"> <@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" /> @@ -48,8 +49,8 @@ package org.apache.arrow.vector; * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); +public final class ${className} extends BaseDataValueVector implements VariableWidthVector{ + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; @@ -66,22 +67,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V private int allocationSizeInBytes = INITIAL_BYTE_COUNT; private int allocationMonitor = 0; - <#if minor.class == "Decimal"> + <#if minor.typeParams??> + <#list minor.typeParams as typeParam> + private final ${typeParam.type} ${typeParam.name}; + </#list> - private final int precision; - private final int scale; - - public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) { + public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { super(name, allocator); this.oAccessor = offsetVector.getAccessor(); this.accessor = new Accessor(); this.mutator = new Mutator(); - this.precision = precision; - this.scale = scale; + <#list minor.typeParams as typeParam> + this.${typeParam.name} = ${typeParam.name}; + </#list> } <#else> - - public ${minor.class}Vector(String name, BufferAllocator allocator) { + public ${className}(String name, BufferAllocator allocator) { super(name, allocator); this.oAccessor = offsetVector.getAccessor(); this.accessor = new Accessor(); @@ -188,10 +189,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${minor.class}Vector) to); + return new TransferImpl((${className}) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${className} target){ target.clear(); this.offsetVector.transferTo(target.offsetVector); target.data = data.transferOwnership(target.allocator).buffer; @@ -199,7 +200,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V clear(); } - public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { + public void splitAndTransferTo(int startIndex, int length, ${className} target) { UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor(); final int startPoint = offsetVectorAccessor.get(startIndex); final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; @@ -214,7 +215,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V target.getMutator().setValueCount(length); } - protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + protected void copyFrom(int fromIndex, int thisIndex, ${className} from){ final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); final int start = fromOffsetVectorAccessor.get(fromIndex); final int end = fromOffsetVectorAccessor.get(fromIndex + 1); @@ -225,7 +226,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len); } - public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public boolean copyFromSafe(int fromIndex, int thisIndex, ${className} from){ final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); final int start = fromOffsetVectorAccessor.get(fromIndex); final int end = fromOffsetVectorAccessor.get(fromIndex + 1); @@ -242,22 +243,18 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } private class TransferImpl implements TransferPair{ - ${minor.class}Vector to; + ${className} to; public TransferImpl(String name, BufferAllocator allocator){ - <#if minor.class == "Decimal"> - to = new ${minor.class}Vector(name, allocator, precision, scale); - <#else> - to = new ${minor.class}Vector(name, allocator); - </#if> + to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>, ${className}.this.${typeParam.name}</#list></#if>); } - public TransferImpl(${minor.class}Vector to){ + public TransferImpl(${className} to){ this.to = to; } @Override - public ${minor.class}Vector getTo(){ + public ${className} getTo(){ return to; } @@ -273,7 +270,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this); + to.copyFromSafe(fromIndex, toIndex, ${className}.this); } } http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 21aa037..3ef1484 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -43,9 +43,13 @@ import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeNanoVector; import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecTZVector; import org.apache.arrow.vector.TimeStampSecVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.UInt1Vector; @@ -61,14 +65,14 @@ import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.schema.ArrowVectorType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.MappingJsonFactory; import com.google.common.base.Objects; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; public class JsonFileReader implements AutoCloseable { private final File inputFile; @@ -278,6 +282,18 @@ public class JsonFileReader implements AutoCloseable { case TIMESTAMPNANO: ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); break; + case TIMESTAMPSECTZ: + ((TimeStampSecTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLITZ: + ((TimeStampMilliTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICROTZ: + ((TimeStampMicroTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANOTZ: + ((TimeStampNanoTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; default: throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType()); } http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 2a0e47b..6591a4b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -38,9 +38,13 @@ import org.apache.arrow.vector.NullableTimeMicroVector; import org.apache.arrow.vector.NullableTimeMilliVector; import org.apache.arrow.vector.NullableTimeNanoVector; import org.apache.arrow.vector.NullableTimeSecVector; +import org.apache.arrow.vector.NullableTimeStampMicroTZVector; import org.apache.arrow.vector.NullableTimeStampMicroVector; +import org.apache.arrow.vector.NullableTimeStampMilliTZVector; import org.apache.arrow.vector.NullableTimeStampMilliVector; +import org.apache.arrow.vector.NullableTimeStampNanoTZVector; import org.apache.arrow.vector.NullableTimeStampNanoVector; +import org.apache.arrow.vector.NullableTimeStampSecTZVector; import org.apache.arrow.vector.NullableTimeStampSecVector; import org.apache.arrow.vector.NullableTinyIntVector; import org.apache.arrow.vector.NullableUInt1Vector; @@ -71,9 +75,13 @@ import org.apache.arrow.vector.complex.impl.TimeMicroWriterImpl; import org.apache.arrow.vector.complex.impl.TimeMilliWriterImpl; import org.apache.arrow.vector.complex.impl.TimeNanoWriterImpl; import org.apache.arrow.vector.complex.impl.TimeSecWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampMicroTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampMilliTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampNanoTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampSecTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl; import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl; import org.apache.arrow.vector.complex.impl.UInt1WriterImpl; @@ -369,11 +377,6 @@ public class Types { }, DECIMAL(null) { @Override - public ArrowType getType() { - throw new UnsupportedOperationException("Cannot get simple type for Decimal type"); - } - - @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableDecimalVector(name, fieldType, allocator); } @@ -440,11 +443,6 @@ public class Types { }, FIXED_SIZE_LIST(null) { @Override - public ArrowType getType() { - throw new UnsupportedOperationException("Cannot get simple type for FixedSizeList type"); - } - - @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); } @@ -467,6 +465,50 @@ public class Types { public FieldWriter getNewFieldWriter(ValueVector vector) { return new UnionWriter((UnionVector) vector); } + }, + TIMESTAMPSECTZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampSecTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampSecTZWriterImpl((NullableTimeStampSecTZVector) vector); + } + }, + TIMESTAMPMILLITZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampMilliTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampMilliTZWriterImpl((NullableTimeStampMilliTZVector) vector); + } + }, + TIMESTAMPMICROTZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampMicroTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampMicroTZWriterImpl((NullableTimeStampMicroTZVector) vector); + } + }, + TIMESTAMPNANOTZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampNanoTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampNanoTZWriterImpl((NullableTimeStampNanoTZVector) vector); + } }; private final ArrowType type; @@ -475,7 +517,10 @@ public class Types { this.type = type; } - public ArrowType getType() { + public final ArrowType getType() { + if (type == null) { + throw new UnsupportedOperationException("Cannot get simple type for type " + name()); + } return type; } @@ -579,18 +624,16 @@ public class Types { } @Override public MinorType visit(Timestamp type) { - if (type.getTimezone() != null) { - throw new IllegalArgumentException("only timezone-less timestamps are supported for now: " + type); - } + String tz = type.getTimezone(); switch (type.getUnit()) { case SECOND: - return MinorType.TIMESTAMPSEC; + return tz == null ? MinorType.TIMESTAMPSEC : MinorType.TIMESTAMPSECTZ; case MILLISECOND: - return MinorType.TIMESTAMPMILLI; + return tz == null ? MinorType.TIMESTAMPMILLI : MinorType.TIMESTAMPMILLITZ; case MICROSECOND: - return MinorType.TIMESTAMPMICRO; + return tz == null ? MinorType.TIMESTAMPMICRO : MinorType.TIMESTAMPMICROTZ; case NANOSECOND: - return MinorType.TIMESTAMPNANO; + return tz == null ? MinorType.TIMESTAMPNANO : MinorType.TIMESTAMPNANOTZ; default: throw new IllegalArgumentException("unknown unit: " + type); } http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 1613936..ede8d65 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -42,15 +42,18 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.ArrowType.Int; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp; import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.DateUtility; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; @@ -592,85 +595,198 @@ public class TestComplexWriter { } @Test - public void timeStampWriters() throws Exception { + public void timeStampSecWriter() throws Exception { // test values - final long expectedNanos = 981173106123456789L; - final long expectedMicros = 981173106123456L; - final long expectedMillis = 981173106123L; final long expectedSecs = 981173106L; final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0); + + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + + { + TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec"); + timeStampSecWriter.setPosition(0); + timeStampSecWriter.writeTimeStampSec(expectedSecs); + } + { + TimeStampSecTZWriter timeStampSecTZWriter = rootWriter.timeStampSecTZ("secTZ", "UTC"); + timeStampSecTZWriter.setPosition(1); + timeStampSecTZWriter.writeTimeStampSecTZ(expectedSecs); + } + // schema + List<Field> children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "sec"); + checkTimestampTZField(children.get(1), "secTZ", "UTC"); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + { + FieldReader secReader = rootReader.reader("sec"); + secReader.setPosition(0); + LocalDateTime secDateTime = secReader.readLocalDateTime(); + Assert.assertEquals(expectedSecDateTime, secDateTime); + long secLong = secReader.readLong(); + Assert.assertEquals(expectedSecs, secLong); + } + { + FieldReader secTZReader = rootReader.reader("secTZ"); + secTZReader.setPosition(1); + long secTZLong = secTZReader.readLong(); + Assert.assertEquals(expectedSecs, secTZLong); + } + } + + @Test + public void timeStampMilliWriters() throws Exception { + // test values + final long expectedMillis = 981173106123L; final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); - final LocalDateTime expectedMicroDateTime = expectedMilliDateTime; - final LocalDateTime expectedNanoDateTime = expectedMilliDateTime; // write MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); + { + TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli"); + timeStampWriter.setPosition(0); + timeStampWriter.writeTimeStampMilli(expectedMillis); + } + String tz = DateUtility.getTimeZone(10); + { + TimeStampMilliTZWriter timeStampTZWriter = rootWriter.timeStampMilliTZ("milliTZ", tz); + timeStampTZWriter.setPosition(0); + timeStampTZWriter.writeTimeStampMilliTZ(expectedMillis); + } + // schema + List<Field> children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "milli"); + checkTimestampTZField(children.get(1), "milliTZ", tz); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + + { + FieldReader milliReader = rootReader.reader("milli"); + milliReader.setPosition(0); + LocalDateTime milliDateTime = milliReader.readLocalDateTime(); + Assert.assertEquals(expectedMilliDateTime, milliDateTime); + long milliLong = milliReader.readLong(); + Assert.assertEquals(expectedMillis, milliLong); + } + { + FieldReader milliTZReader = rootReader.reader("milliTZ"); + milliTZReader.setPosition(0); + long milliTZLong = milliTZReader.readLong(); + Assert.assertEquals(expectedMillis, milliTZLong); + } - TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec"); - timeStampSecWriter.setPosition(0); - timeStampSecWriter.writeTimeStampSec(expectedSecs); + } - TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli"); - timeStampWriter.setPosition(1); - timeStampWriter.writeTimeStampMilli(expectedMillis); + private void checkTimestampField(Field field, String name) { + Assert.assertEquals(name, field.getName()); + Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, field.getType().getTypeID()); + } - TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro"); - timeStampMicroWriter.setPosition(2); - timeStampMicroWriter.writeTimeStampMicro(expectedMicros); + private void checkTimestampTZField(Field field, String name, String tz) { + checkTimestampField(field, name); + Assert.assertEquals(tz, ((Timestamp)field.getType()).getTimezone()); + } - TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano"); - timeStampNanoWriter.setPosition(3); - timeStampNanoWriter.writeTimeStampNano(expectedNanos); + @Test + public void timeStampMicroWriters() throws Exception { + // test values + final long expectedMicros = 981173106123456L; + final LocalDateTime expectedMicroDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); + + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + + { + TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro"); + timeStampMicroWriter.setPosition(0); + timeStampMicroWriter.writeTimeStampMicro(expectedMicros); + } + String tz = DateUtility.getTimeZone(5); + { + TimeStampMicroTZWriter timeStampMicroWriter = rootWriter.timeStampMicroTZ("microTZ", tz); + timeStampMicroWriter.setPosition(1); + timeStampMicroWriter.writeTimeStampMicroTZ(expectedMicros); + } // schema - Field secField = parent.getField().getChildren().get(0).getChildren().get(0); - Assert.assertEquals("sec", secField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, secField.getType().getTypeID()); + List<Field> children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "micro"); + checkTimestampTZField(children.get(1), "microTZ", tz); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + { + FieldReader microReader = rootReader.reader("micro"); + microReader.setPosition(0); + LocalDateTime microDateTime = microReader.readLocalDateTime(); + Assert.assertEquals(expectedMicroDateTime, microDateTime); + long microLong = microReader.readLong(); + Assert.assertEquals(expectedMicros, microLong); + } + { + FieldReader microReader = rootReader.reader("microTZ"); + microReader.setPosition(1); + long microLong = microReader.readLong(); + Assert.assertEquals(expectedMicros, microLong); + } - Field milliField = parent.getField().getChildren().get(0).getChildren().get(1); - Assert.assertEquals("milli", milliField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, milliField.getType().getTypeID()); + } - Field microField = parent.getField().getChildren().get(0).getChildren().get(2); - Assert.assertEquals("micro", microField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, microField.getType().getTypeID()); + @Test + public void timeStampNanoWriters() throws Exception { + // test values + final long expectedNanos = 981173106123456789L; + final LocalDateTime expectedNanoDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); - Field nanoField = parent.getField().getChildren().get(0).getChildren().get(3); - Assert.assertEquals("nano", nanoField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, nanoField.getType().getTypeID()); + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + { + TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano"); + timeStampNanoWriter.setPosition(0); + timeStampNanoWriter.writeTimeStampNano(expectedNanos); + } + String tz = DateUtility.getTimeZone(3); + { + TimeStampNanoTZWriter timeStampNanoWriter = rootWriter.timeStampNanoTZ("nanoTZ", tz); + timeStampNanoWriter.setPosition(0); + timeStampNanoWriter.writeTimeStampNanoTZ(expectedNanos); + } + // schema + List<Field> children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "nano"); + checkTimestampTZField(children.get(1), "nanoTZ", tz); // read MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); - FieldReader secReader = rootReader.reader("sec"); - secReader.setPosition(0); - LocalDateTime secDateTime = secReader.readLocalDateTime(); - Assert.assertEquals(expectedSecDateTime, secDateTime); - long secLong = secReader.readLong(); - Assert.assertEquals(expectedSecs, secLong); - - FieldReader milliReader = rootReader.reader("milli"); - milliReader.setPosition(1); - LocalDateTime milliDateTime = milliReader.readLocalDateTime(); - Assert.assertEquals(expectedMilliDateTime, milliDateTime); - long milliLong = milliReader.readLong(); - Assert.assertEquals(expectedMillis, milliLong); - - FieldReader microReader = rootReader.reader("micro"); - microReader.setPosition(2); - LocalDateTime microDateTime = microReader.readLocalDateTime(); - Assert.assertEquals(expectedMicroDateTime, microDateTime); - long microLong = microReader.readLong(); - Assert.assertEquals(expectedMicros, microLong); - - FieldReader nanoReader = rootReader.reader("nano"); - nanoReader.setPosition(3); - LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); - Assert.assertEquals(expectedNanoDateTime, nanoDateTime); - long nanoLong = nanoReader.readLong(); - Assert.assertEquals(expectedNanos, nanoLong); + { + FieldReader nanoReader = rootReader.reader("nano"); + nanoReader.setPosition(0); + LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); + Assert.assertEquals(expectedNanoDateTime, nanoDateTime); + long nanoLong = nanoReader.readLong(); + Assert.assertEquals(expectedNanos, nanoLong); + } + { + FieldReader nanoReader = rootReader.reader("nanoTZ"); + nanoReader.setPosition(0); + long nanoLong = nanoReader.readLong(); + Assert.assertEquals(expectedNanos, nanoLong); + NullableTimeStampNanoTZHolder h = new NullableTimeStampNanoTZHolder(); + nanoReader.read(h); + Assert.assertEquals(expectedNanos, h.value); + } } @Test @@ -710,4 +826,4 @@ public class TestComplexWriter { innerMap = (JsonStringHashMap<?,?>) object.get(3); assertEquals(2, innerMap.get("a")); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 5cc36a3..63027e6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -37,8 +37,10 @@ import org.apache.arrow.vector.complex.writer.BigIntWriter; import org.apache.arrow.vector.complex.writer.DateMilliWriter; import org.apache.arrow.vector.complex.writer.IntWriter; import org.apache.arrow.vector.complex.writer.TimeMilliWriter; +import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter; import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter; import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; +import org.apache.arrow.vector.util.DateUtility; import org.joda.time.DateTimeZone; import org.joda.time.LocalDateTime; import org.junit.After; @@ -155,18 +157,21 @@ public class BaseFileTest { DateMilliWriter dateWriter = rootWriter.dateMilli("date"); TimeMilliWriter timeWriter = rootWriter.timeMilli("time"); TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli"); + TimeStampMilliTZWriter timeStampMilliTZWriter = rootWriter.timeStampMilliTZ("timestamp-milliTZ", "Europe/Paris"); for (int i = 0; i < count; i++) { LocalDateTime dt = makeDateTimeFromCount(i); // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used dateWriter.setPosition(i); - long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay())); + long dateLong = DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay())); dateWriter.writeDateMilli(dateLong); // Time is a value in milliseconds since midnight, stored as 32-bit integer timeWriter.setPosition(i); timeWriter.writeTimeMilli(dt.getMillisOfDay()); // Timestamp is milliseconds since the epoch, stored as 64-bit integer timeStampMilliWriter.setPosition(i); - timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt)); + timeStampMilliWriter.writeTimeStampMilli(DateUtility.toMillis(dt)); + timeStampMilliTZWriter.setPosition(i); + timeStampMilliTZWriter.writeTimeStampMilliTZ(DateUtility.toMillis(dt)); } writer.setValueCount(count); } @@ -178,11 +183,13 @@ public class BaseFileTest { long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i); LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); - Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected), dateVal); + Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal); long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); Assert.assertEquals(dt, timestampMilliVal); + Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i); + Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal); } } http://git-wip-us.apache.org/repos/asf/arrow/blob/681afabb/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 64f7970..e2dae29 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -83,7 +83,8 @@ public class TestConvert { ))); childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.<Field>of( new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null), - new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of()) + new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of()), + new Field("child5.3", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null) ))); Schema initialSchema = new Schema(childrenBuilder.build()); run(initialSchema);
