Repository: crunch Updated Branches: refs/heads/master 0d31415e4 -> 1252e7f91
CRUNCH-597: Upgrade to Parquet 1.8.1 Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/1252e7f9 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/1252e7f9 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/1252e7f9 Branch: refs/heads/master Commit: 1252e7f9181053132d35d13b0137f997786c79ff Parents: 0d31415 Author: Josh Wills <[email protected]> Authored: Thu Mar 24 09:55:16 2016 -0700 Committer: Josh Wills <[email protected]> Committed: Thu Mar 24 09:55:16 2016 -0700 ---------------------------------------------------------------------- crunch-core/pom.xml | 2 +- .../crunch/io/parquet/AvroParquetFileSourceTargetIT.java | 8 ++++---- .../org/apache/crunch/io/parquet/AvroParquetPipelineIT.java | 4 ++-- .../src/main/java/org/apache/crunch/PipelineCallable.java | 2 +- crunch-core/src/main/java/org/apache/crunch/io/Compress.java | 2 +- .../crunch/io/parquet/AvroParquetFileReaderFactory.java | 6 +++--- .../org/apache/crunch/io/parquet/AvroParquetFileSource.java | 8 ++++---- .../org/apache/crunch/io/parquet/AvroParquetFileTarget.java | 4 ++-- .../crunch/io/parquet/AvroParquetFileReaderFactoryTest.java | 2 +- .../java/org/apache/crunch/SparkAvroParquetPipelineIT.java | 4 ++-- pom.xml | 4 ++-- 11 files changed, 23 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml index 75f7208..603c231 100644 --- a/crunch-core/pom.xml +++ b/crunch-core/pom.xml @@ -46,7 +46,7 @@ under the License. </dependency> <dependency> - <groupId>com.twitter</groupId> + <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java index d75d9da..a14a5e3 100644 --- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java @@ -51,12 +51,12 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import parquet.avro.AvroParquetWriter; +import org.apache.parquet.avro.AvroParquetWriter; import com.google.common.collect.Lists; -import parquet.column.ColumnReader; -import parquet.filter.RecordFilter; -import parquet.filter.UnboundRecordFilter; +import org.apache.parquet.column.ColumnReader; +import org.apache.parquet.filter.RecordFilter; +import org.apache.parquet.filter.UnboundRecordFilter; @SuppressWarnings("serial") public class AvroParquetFileSourceTargetIT implements Serializable { http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java index 055d0d7..62ab1c2 100644 --- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java @@ -43,8 +43,8 @@ import org.apache.hadoop.fs.Path; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import parquet.avro.AvroParquetReader; -import parquet.avro.AvroParquetWriter; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroParquetWriter; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java b/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java index 9f6829e..4ab6df1 100644 --- a/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java +++ b/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java @@ -17,13 +17,13 @@ */ package org.apache.crunch; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import parquet.Preconditions; import java.util.Map; import java.util.concurrent.Callable; http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/Compress.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/Compress.java b/crunch-core/src/main/java/org/apache/crunch/io/Compress.java index 881b017..bbf09e5 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/Compress.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/Compress.java @@ -53,7 +53,7 @@ public class Compress { public static <T extends Target> T snappy(T target) { Class<? extends CompressionCodec> snappyCodec = org.apache.hadoop.io.compress.SnappyCodec.class; if (target instanceof AvroParquetFileTarget || target instanceof AvroParquetFileSourceTarget) { - snappyCodec = parquet.hadoop.codec.SnappyCodec.class; + snappyCodec = org.apache.parquet.hadoop.codec.SnappyCodec.class; } return (T) compress(target, snappyCodec) .outputConf(AvroJob.OUTPUT_CODEC, DataFileConstants.SNAPPY_CODEC); http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java index c193563..0b631a8 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java @@ -30,9 +30,9 @@ import org.apache.crunch.types.avro.AvroType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import parquet.avro.AvroReadSupport; -import parquet.hadoop.ParquetReader; -import parquet.schema.MessageType; +import org.apache.parquet.avro.AvroReadSupport; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.schema.MessageType; class AvroParquetFileReaderFactory<T> implements FileReaderFactory<T> { http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java index 18b2928..09703bd 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java @@ -40,10 +40,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.util.ReflectionUtils; -import parquet.avro.AvroParquetInputFormat; -import parquet.avro.AvroReadSupport; -import parquet.filter.UnboundRecordFilter; -import parquet.hadoop.ParquetInputSplit; +import org.apache.parquet.avro.AvroParquetInputFormat; +import org.apache.parquet.avro.AvroReadSupport; +import org.apache.parquet.filter.UnboundRecordFilter; +import org.apache.parquet.hadoop.ParquetInputSplit; public class AvroParquetFileSource<T extends IndexedRecord> extends FileSourceImpl<T> implements ReadableSource<T> { http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java index 34ee14b..2698053 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java @@ -33,8 +33,8 @@ import org.apache.crunch.types.avro.AvroType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Job; -import parquet.avro.AvroWriteSupport; -import parquet.hadoop.ParquetOutputFormat; +import org.apache.parquet.avro.AvroWriteSupport; +import org.apache.parquet.hadoop.ParquetOutputFormat; import java.util.Map; http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java index 9f5ff70..d2ad5bc 100644 --- a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java @@ -41,7 +41,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import parquet.avro.AvroParquetWriter; +import org.apache.parquet.avro.AvroParquetWriter; public class AvroParquetFileReaderFactoryTest { http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java ---------------------------------------------------------------------- diff --git a/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java b/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java index f5e2c25..a86718e 100644 --- a/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java +++ b/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java @@ -37,8 +37,8 @@ import org.apache.hadoop.fs.Path; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import parquet.avro.AvroParquetReader; -import parquet.avro.AvroParquetWriter; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroParquetWriter; import java.io.File; import java.io.FileOutputStream; http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7389c76..78ad9b7 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,7 @@ under the License. <commons-cli.version>1.2</commons-cli.version> <avro.version>1.7.7</avro.version> <hive.version>0.13.1</hive.version> - <parquet.version>1.4.3</parquet.version> + <parquet.version>1.8.1</parquet.version> <javassist.version>3.16.1-GA</javassist.version> <jackson.version>1.8.8</jackson.version> <protobuf-java.version>2.5.0</protobuf-java.version> @@ -242,7 +242,7 @@ under the License. </dependency> <dependency> - <groupId>com.twitter</groupId> + <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>${parquet.version}</version> <exclusions>
