This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 3e8975bc404 [MINOR][SQL] Remove ColumnIOUtil 3e8975bc404 is described below commit 3e8975bc404d985e3f48cc249b1c26baf9dba601 Author: panbingkun <pbk1...@gmail.com> AuthorDate: Tue Aug 16 08:35:17 2022 -0500 [MINOR][SQL] Remove ColumnIOUtil ### What changes were proposed in this pull request? This PR aim to remove redundant ColumnIOUtil. ### Why are the changes needed? Complete todo. <img width="464" alt="image" src="https://user-images.githubusercontent.com/15246973/184788406-abcf6794-e623-4e64-8dc7-2ff1b05dcbed.png"> reason: from parquet version 1.12.3, methods below are public > 1.ColumnIO.getDefinitionLevel: https://github.com/apache/parquet-mr/blob/apache-parquet-1.12.3/parquet-column/src/main/java/org/apache/parquet/io/ColumnIO.java#L84-L86 > 2.ColumnIO.getRepetitionLevel: https://github.com/apache/parquet-mr/blob/apache-parquet-1.12.3/parquet-column/src/main/java/org/apache/parquet/io/ColumnIO.java#L77-L79 > 3.ColumnIO.getFieldPath: https://github.com/apache/parquet-mr/blob/apache-parquet-1.12.3/parquet-column/src/main/java/org/apache/parquet/io/ColumnIO.java#L50-L52 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA. Closes #37529 from panbingkun/remove_ColumnIOUtil. Authored-by: panbingkun <pbk1...@gmail.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- .../java/org/apache/parquet/io/ColumnIOUtil.java | 40 ---------------------- .../datasources/parquet/ParquetColumn.scala | 13 ++++--- 2 files changed, 6 insertions(+), 47 deletions(-) diff --git a/sql/core/src/main/java/org/apache/parquet/io/ColumnIOUtil.java b/sql/core/src/main/java/org/apache/parquet/io/ColumnIOUtil.java deleted file mode 100644 index d4f93e54cac..00000000000 --- a/sql/core/src/main/java/org/apache/parquet/io/ColumnIOUtil.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.parquet.io; - -/** - * This is a workaround since methods below are not public in {@link ColumnIO}. - * - * TODO(SPARK-36511): we should remove this once PARQUET-2050 and PARQUET-2083 are released with - * Parquet 1.13. - */ -public class ColumnIOUtil { - private ColumnIOUtil() {} - - public static int getDefinitionLevel(ColumnIO column) { - return column.getDefinitionLevel(); - } - - public static int getRepetitionLevel(ColumnIO column) { - return column.getRepetitionLevel(); - } - - public static String[] getFieldPath(ColumnIO column) { - return column.getFieldPath(); - } -} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumn.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumn.scala index 5d05d6a6759..cbe6eb99a98 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumn.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumn.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.datasources.parquet import org.apache.parquet.column.ColumnDescriptor -import org.apache.parquet.io.ColumnIOUtil import org.apache.parquet.io.GroupColumnIO import org.apache.parquet.io.PrimitiveColumnIO import org.apache.parquet.schema.Type.Repetition @@ -42,14 +41,14 @@ case class ParquetColumn( object ParquetColumn { def apply(sparkType: DataType, io: PrimitiveColumnIO): ParquetColumn = { - this(sparkType, Some(io.getColumnDescriptor), ColumnIOUtil.getRepetitionLevel(io), - ColumnIOUtil.getDefinitionLevel(io), io.getType.isRepetition(Repetition.REQUIRED), - ColumnIOUtil.getFieldPath(io), Seq.empty) + this(sparkType, Some(io.getColumnDescriptor), io.getRepetitionLevel, + io.getDefinitionLevel, io.getType.isRepetition(Repetition.REQUIRED), + io.getFieldPath, Seq.empty) } def apply(sparkType: DataType, io: GroupColumnIO, children: Seq[ParquetColumn]): ParquetColumn = { - this(sparkType, None, ColumnIOUtil.getRepetitionLevel(io), - ColumnIOUtil.getDefinitionLevel(io), io.getType.isRepetition(Repetition.REQUIRED), - ColumnIOUtil.getFieldPath(io), children) + this(sparkType, None, io.getRepetitionLevel, + io.getDefinitionLevel, io.getType.isRepetition(Repetition.REQUIRED), + io.getFieldPath, children) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org