Repository: arrow Updated Branches: refs/heads/master 1a6d135be -> 316c63dba
ARROW-824: Date and Time Vectors should reflect timezone-less semantics The current java vector support the timezone less version of time related types but in an incomplete way. This change fixes it and clarifies what the vectors implementation do. We'll need separate vectors or adapt those to deal will timezone aware time types. Author: Julien Le Dem <[email protected]> Closes #568 from julienledem/ARROW-824 and squashes the following commits: 3528ad8 [Julien Le Dem] add license e37385c [Julien Le Dem] centralize LocalDateTime.toMillis bdac7ff [Julien Le Dem] make integration tests output more readable b0da88c [Julien Le Dem] fix failing integration test 61518ec [Julien Le Dem] improve travis integration ec19e7d [Julien Le Dem] ARROW-824: Date and Time Vectors should reflect timezone-less semantics Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/316c63db Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/316c63db Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/316c63db Branch: refs/heads/master Commit: 316c63dbaec8e5df33d0cf0fa78d38ac8cc375b8 Parents: 1a6d135 Author: Julien Le Dem <[email protected]> Authored: Fri May 5 22:05:46 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Fri May 5 22:05:46 2017 -0400 ---------------------------------------------------------------------- ci/travis_script_integration.sh | 3 +- integration/integration_test.py | 27 +++++++++++------- .../src/main/codegen/data/ValueVectorTypes.tdd | 14 ++++----- .../src/main/codegen/includes/vv_imports.ftl | 1 + .../codegen/templates/AbstractFieldReader.java | 6 ++-- .../src/main/codegen/templates/ArrowType.java | 1 + .../src/main/codegen/templates/BaseReader.java | 5 ++-- .../main/codegen/templates/ComplexReaders.java | 7 +++++ .../main/codegen/templates/ComplexWriters.java | 3 ++ .../codegen/templates/FixedValueVectors.java | 18 ++++-------- .../codegen/templates/HolderReaderImpl.java | 12 ++++---- .../src/main/codegen/templates/NullReader.java | 6 ++-- .../src/main/codegen/templates/UnionReader.java | 6 ++-- .../src/main/codegen/templates/UnionVector.java | 1 + .../main/codegen/templates/ValueHolders.java | 3 ++ .../org/apache/arrow/vector/types/Types.java | 11 ++++--- .../apache/arrow/vector/util/DateUtility.java | 10 +++++++ .../main/java/org/joda/time/LocalDateTimes.java | 30 ++++++++++++++++++++ .../complex/writer/TestComplexWriter.java | 19 ++++++------- .../apache/arrow/vector/file/BaseFileTest.java | 20 ++++++------- .../apache/arrow/vector/pojo/TestConvert.java | 2 +- 21 files changed, 134 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/ci/travis_script_integration.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index 56f5ab7..6e93ed7 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh pushd $ARROW_JAVA_DIR -mvn package +echo "mvn package" +mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false) popd http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/integration/integration_test.py ---------------------------------------------------------------------- diff --git a/integration/integration_test.py b/integration/integration_test.py index 661f5c9..6466469 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -544,7 +544,8 @@ class JSONRecordBatch(object): class JSONFile(object): - def __init__(self, schema, batches): + def __init__(self, name, schema, batches): + self.name = name self.schema = schema self.batches = batches @@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True): raise TypeError(dtype) -def _generate_file(fields, batch_sizes): +def _generate_file(name, fields, batch_sizes): schema = JSONSchema(fields) batches = [] for size in batch_sizes: @@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes): batches.append(JSONRecordBatch(size, columns)) - return JSONFile(schema, batches) + return JSONFile(name, schema, batches) def generate_primitive_case(batch_sizes): @@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes): fields.append(get_field(type_ + "_nullable", type_, True)) fields.append(get_field(type_ + "_nonnullable", type_, False)) - return _generate_file(fields, batch_sizes) + return _generate_file("primitive", fields, batch_sizes) def generate_datetime_case(): @@ -619,11 +620,11 @@ def generate_datetime_case(): TimestampType('f7', 'ms'), TimestampType('f8', 'us'), TimestampType('f9', 'ns'), - TimestampType('f10', 'ms', tz='America/New_York') + TimestampType('f10', 'ms', tz=None) ] batch_sizes = [7, 10] - return _generate_file(fields, batch_sizes) + return _generate_file("datetime", fields, batch_sizes) def generate_nested_case(): @@ -637,7 +638,7 @@ def generate_nested_case(): ] batch_sizes = [7, 10] - return _generate_file(fields, batch_sizes) + return _generate_file("nested", fields, batch_sizes) def get_generated_json_files(): @@ -655,7 +656,7 @@ def get_generated_json_files(): generated_paths = [] for file_obj in file_objs: - out_path = os.path.join(temp_dir, guid() + '.json') + out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json') file_obj.write(out_path) generated_paths.append(out_path) @@ -684,11 +685,15 @@ class IntegrationRunner(object): consumer.name)) for json_path in self.json_files: + print('=====================================================================================') print('Testing file {0}'.format(json_path)) + print('=====================================================================================') + + name = os.path.splitext(os.path.basename(json_path))[0] # Make the random access file print('-- Creating binary inputs') - producer_file_path = os.path.join(self.temp_dir, guid()) + producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow') producer.json_to_file(json_path, producer_file_path) # Validate the file @@ -696,8 +701,8 @@ class IntegrationRunner(object): consumer.validate(json_path, producer_file_path) print('-- Validating stream') - producer_stream_path = os.path.join(self.temp_dir, guid()) - consumer_file_path = os.path.join(self.temp_dir, guid()) + producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream') + consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow') producer.file_to_stream(producer_file_path, producer_stream_path) consumer.stream_to_file(producer_stream_path, http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/data/ValueVectorTypes.tdd ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index d472b55..ca6d9ec 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -59,7 +59,7 @@ { class: "DateDay" }, { class: "IntervalYear", javaType: "int", friendlyType: "Period" }, { class: "TimeSec" }, - { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" } + { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" } ] }, { @@ -71,12 +71,12 @@ minor: [ { class: "BigInt"}, { class: "UInt8" }, - { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], }, - { class: "DateMilli", javaType: "long", friendlyType: "DateTime" }, - { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, - { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, - { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, - { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, + { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], }, + { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, + { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, { class: "TimeMicro" }, { class: "TimeNano" } ] http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/includes/vv_imports.ftl ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl index 9b4b79b..e723e7d 100644 --- a/java/vector/src/main/codegen/includes/vv_imports.ftl +++ b/java/vector/src/main/codegen/includes/vv_imports.ftl @@ -57,6 +57,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import org.joda.time.DateTime; +import org.joda.time.LocalDateTime; import org.joda.time.Period; http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/AbstractFieldReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java index 79d4c12..b16ee16 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -26,8 +26,8 @@ package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> -/* - * This class is generated using freemarker and the ${.template_name} template. +/** + * Source code generated using FreeMarker template ${.template_name} */ @SuppressWarnings("unused") abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{ @@ -51,7 +51,7 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe } <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", - "Character", "DateTime", "Period", "Double", "Float", + "Character", "LocalDateTime", "Period", "Double", "Float", "Text", "String", "Byte", "Short", "byte[]"] as friendlyType> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ArrowType.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java index dc99aad..9374630 100644 --- a/java/vector/src/main/codegen/templates/ArrowType.java +++ b/java/vector/src/main/codegen/templates/ArrowType.java @@ -38,6 +38,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; /** * Arrow types + * Source code generated using FreeMarker template ${.template_name} **/ @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/BaseReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java index 72fea58..ea3286e 100644 --- a/java/vector/src/main/codegen/templates/BaseReader.java +++ b/java/vector/src/main/codegen/templates/BaseReader.java @@ -26,8 +26,9 @@ package org.apache.arrow.vector.complex.reader; <#include "/@includes/vv_imports.ftl" /> - - +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public interface BaseReader extends Positionable{ Field getField(); http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ComplexReaders.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index d537445..38cd1bf 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -47,6 +47,9 @@ package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public class ${name}ReaderImpl extends AbstractFieldReader { @@ -123,12 +126,16 @@ public class ${name}ReaderImpl extends AbstractFieldReader { package org.apache.arrow.vector.complex.reader; <#include "/@includes/vv_imports.ftl" /> +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public interface ${name}Reader extends BaseReader{ public void read(${minor.class?cap_first}Holder h); public void read(Nullable${minor.class?cap_first}Holder h); public Object readObject(); + // read friendly type public ${friendlyType} read${safeType}(); public boolean isSet(); public void copyAsValue(${minor.class}Writer writer); http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ComplexWriters.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 3457545..c23b89d 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -139,6 +139,9 @@ public class ${eName}WriterImpl extends AbstractFieldWriter { package org.apache.arrow.vector.complex.writer; <#include "/@includes/vv_imports.ftl" /> +/* + * This class is generated using FreeMarker on the ${.template_name} template. + */ @SuppressWarnings("unused") public interface ${eName}Writer extends BaseWriter { public void write(${minor.class}Holder h); http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java index 05faaae..f403ecf 100644 --- a/java/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java @@ -495,16 +495,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F <#elseif minor.class == "DateMilli"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); return date; } <#elseif minor.class == "TimeMilli"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); return time; } @@ -512,16 +510,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public ${friendlyType} getObject(int index) { long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index)); - org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC); return date; } <#elseif minor.class == "TimeStampMilli"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); return date; } @@ -530,8 +526,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public ${friendlyType} getObject(int index) { // value is truncated when converting microseconds to milliseconds in order to use DateTime type long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index)); - org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC); return date; } @@ -540,8 +535,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public ${friendlyType} getObject(int index) { // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index)); - org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return date; } http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/HolderReaderImpl.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java index d66577b..e990fcc 100644 --- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java +++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java @@ -84,7 +84,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader { } -@Override + @Override public void read(${name}Holder h) { <#list fields as field> h.${field.name} = holder.${field.name}; @@ -99,7 +99,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader { h.isSet = isSet() ? 1 : 0; } - + // read friendly type @Override public ${friendlyType} read${safeType}(){ <#if nullMode == "Nullable"> @@ -114,15 +114,15 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader { byte[] value = new byte [length]; holder.buffer.getBytes(holder.start, value, 0, length); -<#if minor.class == "VarBinary"> + <#if minor.class == "VarBinary"> return value; -<#elseif minor.class == "Var16Char"> + <#elseif minor.class == "Var16Char"> return new String(value); -<#elseif minor.class == "VarChar"> + <#elseif minor.class == "VarChar"> Text text = new Text(); text.set(value); return text; -</#if> + </#if> <#elseif minor.class == "Interval"> Period p = new Period(); http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/NullReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java index ba0c088..7c75b3a 100644 --- a/java/vector/src/main/codegen/templates/NullReader.java +++ b/java/vector/src/main/codegen/templates/NullReader.java @@ -29,7 +29,9 @@ package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> - +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public class NullReader extends AbstractBaseReader implements FieldReader{ @@ -127,7 +129,7 @@ public class NullReader extends AbstractBaseReader implements FieldReader{ } <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", - "Character", "DateTime", "Period", "Double", "Float", + "Character", "LocalDateTime", "Period", "Double", "Float", "Text", "String", "Byte", "Short", "byte[]"] as friendlyType> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/UnionReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java index c56e95c..0b5a209 100644 --- a/java/vector/src/main/codegen/templates/UnionReader.java +++ b/java/vector/src/main/codegen/templates/UnionReader.java @@ -28,7 +28,9 @@ import org.apache.arrow.vector.types.Types.MinorType; package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> - +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public class UnionReader extends AbstractFieldReader { @@ -122,7 +124,7 @@ public class UnionReader extends AbstractFieldReader { } <#list ["Object", "Integer", "Long", "Boolean", - "Character", "DateTime", "Double", "Float", + "Character", "LocalDateTime", "Double", "Float", "Text", "Byte", "Short", "byte[]"] as friendlyType> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/UnionVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index d70cbae..9d5dee5 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -50,6 +50,7 @@ import static org.apache.arrow.vector.types.UnionMode.Sparse; * * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup * each time the vector is accessed. + * Source code generated using FreeMarker template ${.template_name} */ public class UnionVector implements FieldVector { http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ValueHolders.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java index d744c52..a474b69 100644 --- a/java/vector/src/main/codegen/templates/ValueHolders.java +++ b/java/vector/src/main/codegen/templates/ValueHolders.java @@ -29,6 +29,9 @@ package org.apache.arrow.vector.holders; <#include "/@includes/vv_imports.ftl" /> +/** + * Source code generated using FreeMarker template ${.template_name} + */ public final class ${className} implements ValueHolder{ <#if mode.name == "Repeated"> http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 6023f1c..d5076d8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -241,7 +241,7 @@ public class Types { } }, // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. - TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) { + TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampSecVector(name, fieldType, allocator); @@ -253,7 +253,7 @@ public class Types { } }, // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC. - TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC")) { + TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampMilliVector(name, fieldType, allocator); @@ -265,7 +265,7 @@ public class Types { } }, // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. - TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")) { + TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampMicroVector(name, fieldType, allocator); @@ -277,7 +277,7 @@ public class Types { } }, // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC. - TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")) { + TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampNanoVector(name, fieldType, allocator); @@ -580,6 +580,9 @@ public class Types { } @Override public MinorType visit(Timestamp type) { + if (type.getTimezone() != null) { + throw new IllegalArgumentException("only timezone-less timestamps are supported for now: " + type); + } switch (type.getUnit()) { case SECOND: return MinorType.TIMESTAMPSEC; http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java index 1f8ce06..8aad417 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java @@ -18,6 +18,9 @@ package org.apache.arrow.vector.util; +import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; +import org.joda.time.LocalDateTimes; import org.joda.time.Period; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -679,4 +682,11 @@ public class DateUtility { (period.getMillis()); } + public static long toMillis(LocalDateTime localDateTime) { + return LocalDateTimes.getLocalMillis(localDateTime); + } + + public static int toMillisOfDay(final LocalDateTime localDateTime) { + return localDateTime.toDateTime(DateTimeZone.UTC).millisOfDay().get(); + } } http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/java/org/joda/time/LocalDateTimes.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/joda/time/LocalDateTimes.java b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java new file mode 100644 index 0000000..e4f999e --- /dev/null +++ b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.joda.time; + +/** + * Workaround to access package protected fields in JODA + * + */ +public class LocalDateTimes { + + public static long getLocalMillis(LocalDateTime localDateTime) { + return localDateTime.getLocalMillis(); + } + +} http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 99ba19b..aba65db 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -56,8 +56,7 @@ import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; import org.junit.Assert; import org.junit.Test; @@ -602,10 +601,10 @@ public class TestComplexWriter { final long expectedMicros = 981173106123456L; final long expectedMillis = 981173106123L; final long expectedSecs = 981173106L; - final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault()); - final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault()); - final DateTime expectedMicroDateTime = expectedMilliDateTime; - final DateTime expectedNanoDateTime = expectedMilliDateTime; + final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0); + final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); + final LocalDateTime expectedMicroDateTime = expectedMilliDateTime; + final LocalDateTime expectedNanoDateTime = expectedMilliDateTime; // write MapVector parent = new MapVector("parent", allocator, null); @@ -650,28 +649,28 @@ public class TestComplexWriter { FieldReader secReader = rootReader.reader("sec"); secReader.setPosition(0); - DateTime secDateTime = secReader.readDateTime(); + LocalDateTime secDateTime = secReader.readLocalDateTime(); Assert.assertEquals(expectedSecDateTime, secDateTime); long secLong = secReader.readLong(); Assert.assertEquals(expectedSecs, secLong); FieldReader milliReader = rootReader.reader("milli"); milliReader.setPosition(1); - DateTime milliDateTime = milliReader.readDateTime(); + LocalDateTime milliDateTime = milliReader.readLocalDateTime(); Assert.assertEquals(expectedMilliDateTime, milliDateTime); long milliLong = milliReader.readLong(); Assert.assertEquals(expectedMillis, milliLong); FieldReader microReader = rootReader.reader("micro"); microReader.setPosition(2); - DateTime microDateTime = microReader.readDateTime(); + LocalDateTime microDateTime = microReader.readLocalDateTime(); Assert.assertEquals(expectedMicroDateTime, microDateTime); long microLong = microReader.readLong(); Assert.assertEquals(expectedMicros, microLong); FieldReader nanoReader = rootReader.reader("nano"); nanoReader.setPosition(3); - DateTime nanoDateTime = nanoReader.readDateTime(); + LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); Assert.assertEquals(expectedNanoDateTime, nanoDateTime); long nanoLong = nanoReader.readLong(); Assert.assertEquals(expectedNanos, nanoLong); http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 5ca083a..5cc36a3 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -39,8 +39,8 @@ import org.apache.arrow.vector.complex.writer.IntWriter; import org.apache.arrow.vector.complex.writer.TimeMilliWriter; import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter; import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; -import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -144,8 +144,8 @@ public class BaseFileTest { } } - private DateTime makeDateTimeFromCount(int i) { - return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC); + private LocalDateTime makeDateTimeFromCount(int i) { + return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i); } protected void writeDateTimeData(int count, NullableMapVector parent) { @@ -156,17 +156,17 @@ public class BaseFileTest { TimeMilliWriter timeWriter = rootWriter.timeMilli("time"); TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli"); for (int i = 0; i < count; i++) { - DateTime dt = makeDateTimeFromCount(i); + LocalDateTime dt = makeDateTimeFromCount(i); // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used dateWriter.setPosition(i); - long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis(); + long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay())); dateWriter.writeDateMilli(dateLong); // Time is a value in milliseconds since midnight, stored as 32-bit integer timeWriter.setPosition(i); timeWriter.writeTimeMilli(dt.getMillisOfDay()); // Timestamp is milliseconds since the epoch, stored as 64-bit integer timeStampMilliWriter.setPosition(i); - timeStampMilliWriter.writeTimeStampMilli(dt.getMillis()); + timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt)); } writer.setValueCount(count); } @@ -176,13 +176,13 @@ public class BaseFileTest { printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i); - DateTime dt = makeDateTimeFromCount(i); - DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); - Assert.assertEquals(dateExpected.getMillis(), dateVal); + LocalDateTime dt = makeDateTimeFromCount(i); + LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); + Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected), dateVal); long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); - Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal)); + Assert.assertEquals(dt, timestampMilliVal); } } http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 824c62a..f9c8f72 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -81,7 +81,7 @@ public class TestConvert { new Field("child4.1", true, Utf8.INSTANCE, null) ))); childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of( - new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null), + new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null), new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of()) ))); Schema initialSchema = new Schema(childrenBuilder.build());
