Repository: spark Updated Branches: refs/heads/master 4775eb414 -> c5eb1df72
[SPARK-16814][SQL] Fix deprecated parquet constructor usage ## What changes were proposed in this pull request? Replace deprecated ParquetWriter with the new builders ## How was this patch tested? Existing tests Author: Holden Karau <hol...@us.ibm.com> Closes #14419 from holdenk/SPARK-16814-fix-deprecated-parquet-constructor-usage. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c5eb1df7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c5eb1df7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c5eb1df7 Branch: refs/heads/master Commit: c5eb1df72fea2ecc71369f13416a8aee040b55d2 Parents: 4775eb4 Author: Holden Karau <hol...@us.ibm.com> Authored: Wed Aug 3 17:08:51 2016 -0700 Committer: Sean Owen <so...@cloudera.com> Committed: Wed Aug 3 17:08:51 2016 -0700 ---------------------------------------------------------------------- .../parquet/ParquetAvroCompatibilitySuite.scala | 5 +++-- .../parquet/ParquetCompatibilityTest.scala | 14 ++++++++++++-- .../datasources/parquet/ParquetIOSuite.scala | 16 ++++++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c5eb1df7/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala index 6509e04..1b99fbe 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala @@ -27,6 +27,7 @@ import org.apache.avro.Schema import org.apache.avro.generic.IndexedRecord import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetWriter +import org.apache.parquet.hadoop.ParquetWriter import org.apache.spark.sql.Row import org.apache.spark.sql.execution.datasources.parquet.test.avro._ @@ -35,14 +36,14 @@ import org.apache.spark.sql.test.SharedSQLContext class ParquetAvroCompatibilitySuite extends ParquetCompatibilityTest with SharedSQLContext { private def withWriter[T <: IndexedRecord] (path: String, schema: Schema) - (f: AvroParquetWriter[T] => Unit): Unit = { + (f: ParquetWriter[T] => Unit): Unit = { logInfo( s"""Writing Avro records with the following Avro schema into Parquet file: | |${schema.toString(true)} """.stripMargin) - val writer = new AvroParquetWriter[T](new Path(path), schema) + val writer = AvroParquetWriter.builder[T](new Path(path)).withSchema(schema).build() try f(writer) finally writer.close() } http://git-wip-us.apache.org/repos/asf/spark/blob/c5eb1df7/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala index 57cd70e..a43a856 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala @@ -119,8 +119,18 @@ private[sql] object ParquetCompatibilityTest { metadata: Map[String, String], recordWriters: (RecordConsumer => Unit)*): Unit = { val messageType = MessageTypeParser.parseMessageType(schema) - val writeSupport = new DirectWriteSupport(messageType, metadata) - val parquetWriter = new ParquetWriter[RecordConsumer => Unit](new Path(path), writeSupport) + val testWriteSupport = new DirectWriteSupport(messageType, metadata) + /** + * Provide a builder for constructing a parquet writer - after PARQUET-248 directly constructing + * the writer is deprecated and should be done through a builder. The default builders include + * Avro - but for raw Parquet writing we must create our own builder. + */ + class ParquetWriterBuilder() extends + ParquetWriter.Builder[RecordConsumer => Unit, ParquetWriterBuilder](new Path(path)) { + override def getWriteSupport(conf: Configuration) = testWriteSupport + override def self() = this + } + val parquetWriter = new ParquetWriterBuilder().build() try recordWriters.foreach(parquetWriter.write) finally parquetWriter.close() } } http://git-wip-us.apache.org/repos/asf/spark/blob/c5eb1df7/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala index fc9ce6b..0f74094 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala @@ -325,8 +325,20 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext { |} """.stripMargin) - val writeSupport = new TestGroupWriteSupport(schema) - val writer = new ParquetWriter[Group](path, writeSupport) + val testWriteSupport = new TestGroupWriteSupport(schema) + /** + * Provide a builder for constructing a parquet writer - after PARQUET-248 directly + * constructing the writer is deprecated and should be done through a builder. The default + * builders include Avro - but for raw Parquet writing we must create our own builder. + */ + class ParquetWriterBuilder() extends + ParquetWriter.Builder[Group, ParquetWriterBuilder](path) { + override def getWriteSupport(conf: Configuration) = testWriteSupport + + override def self() = this + } + + val writer = new ParquetWriterBuilder().build() (0 until 10).foreach { i => val record = new SimpleGroup(schema) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
