Repository: parquet-mr Updated Branches: refs/heads/master 30ee10d27 -> c38386d6b
PARQUET-393: Update to parquet-format 2.3.1. Author: Ryan Blue <[email protected]> Closes #303 from rdblue/PARQUET-393-update-parquet-format-version and squashes the following commits: 0e4c798 [Ryan Blue] PARQUET-393: Add TIME_MICROS and TIMESTAMP_MICROS. ca4a741 [Ryan Blue] PARQUET-393: Update to parquet-format 2.3.1. Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/c38386d6 Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/c38386d6 Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/c38386d6 Branch: refs/heads/master Commit: c38386d6b5622915a2d42d989c56d37f17c673d6 Parents: 30ee10d Author: Ryan Blue <[email protected]> Authored: Thu Jan 28 17:33:08 2016 -0800 Committer: Julien Le Dem <[email protected]> Committed: Thu Jan 28 17:33:08 2016 -0800 ---------------------------------------------------------------------- .../main/java/org/apache/parquet/schema/OriginalType.java | 2 ++ .../src/main/java/org/apache/parquet/schema/Types.java | 2 ++ .../java/org/apache/parquet/schema/TestTypeBuilders.java | 4 ++-- .../parquet/format/converter/ParquetMetadataConverter.java | 9 ++++++++- pom.xml | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/c38386d6/parquet-column/src/main/java/org/apache/parquet/schema/OriginalType.java ---------------------------------------------------------------------- diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/OriginalType.java b/parquet-column/src/main/java/org/apache/parquet/schema/OriginalType.java index 0ea89ba..77acc54 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/OriginalType.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/OriginalType.java @@ -27,7 +27,9 @@ public enum OriginalType { DECIMAL, DATE, TIME_MILLIS, + TIME_MICROS, TIMESTAMP_MILLIS, + TIMESTAMP_MICROS, UINT_8, UINT_16, UINT_32, http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/c38386d6/parquet-column/src/main/java/org/apache/parquet/schema/Types.java ---------------------------------------------------------------------- diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java index 947a845..4e7e881 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java @@ -424,7 +424,9 @@ public class Types { Preconditions.checkState(primitiveType == PrimitiveTypeName.INT32, originalType.toString() + " can only annotate INT32"); break; + case TIME_MICROS: case TIMESTAMP_MILLIS: + case TIMESTAMP_MICROS: case UINT_64: case INT_64: Preconditions.checkState(primitiveType == PrimitiveTypeName.INT64, http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/c38386d6/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java ---------------------------------------------------------------------- diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java index 5fac7bc..0c39ef2 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java @@ -509,7 +509,7 @@ public class TestTypeBuilders { @Test public void testInt64Annotations() { OriginalType[] types = new OriginalType[] { - TIMESTAMP_MILLIS, UINT_64, INT_64}; + TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, UINT_64, INT_64}; for (OriginalType logicalType : types) { PrimitiveType expected = new PrimitiveType(REQUIRED, INT64, "col", logicalType); PrimitiveType date = Types.required(INT64).as(logicalType).named("col"); @@ -520,7 +520,7 @@ public class TestTypeBuilders { @Test public void testInt64AnnotationsRejectNonInt64() { OriginalType[] types = new OriginalType[] { - TIMESTAMP_MILLIS, UINT_64, INT_64}; + TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, UINT_64, INT_64}; for (final OriginalType logicalType : types) { PrimitiveTypeName[] nonInt64 = new PrimitiveTypeName[]{ BOOLEAN, INT32, INT96, DOUBLE, FLOAT, BINARY http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/c38386d6/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index 6821bbf..b373bfb 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -36,7 +36,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.parquet.CorruptStatistics; import org.apache.parquet.Log; import org.apache.parquet.hadoop.metadata.ColumnPath; @@ -336,8 +335,12 @@ public class ParquetMetadataConverter { return OriginalType.DATE; case TIME_MILLIS: return OriginalType.TIME_MILLIS; + case TIME_MICROS: + return OriginalType.TIME_MICROS; case TIMESTAMP_MILLIS: return OriginalType.TIMESTAMP_MILLIS; + case TIMESTAMP_MICROS: + return OriginalType.TIMESTAMP_MICROS; case INTERVAL: return OriginalType.INTERVAL; case INT_8: @@ -384,8 +387,12 @@ public class ParquetMetadataConverter { return ConvertedType.DATE; case TIME_MILLIS: return ConvertedType.TIME_MILLIS; + case TIME_MICROS: + return ConvertedType.TIME_MICROS; case TIMESTAMP_MILLIS: return ConvertedType.TIMESTAMP_MILLIS; + case TIMESTAMP_MICROS: + return ConvertedType.TIMESTAMP_MICROS; case INTERVAL: return ConvertedType.INTERVAL; case INT_8: http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/c38386d6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3588de5..94d7a02 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ <shade.prefix>shaded.parquet</shade.prefix> <hadoop.version>1.1.0</hadoop.version> <cascading.version>2.5.3</cascading.version> - <parquet.format.version>2.3.0-incubating</parquet.format.version> + <parquet.format.version>2.3.1</parquet.format.version> <previous.version>1.7.0</previous.version> <thrift.executable>thrift</thrift.executable> <scala.version>2.10.4</scala.version>
