cloud-fan closed pull request #23491: [SPARK-26571][SQL] Update Hive Serde
mapping with canonical name of Parquet and Orc FileFormat
URL: https://github.com/apache/spark/pull/23491
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
index eca612f06f9bb..bd25a6437033c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
@@ -74,8 +74,10 @@ object HiveSerDe {
def sourceToSerDe(source: String): Option[HiveSerDe] = {
val key = source.toLowerCase(Locale.ROOT) match {
case s if s.startsWith("org.apache.spark.sql.parquet") => "parquet"
+ case s if
s.startsWith("org.apache.spark.sql.execution.datasources.parquet") => "parquet"
case s if s.startsWith("org.apache.spark.sql.orc") => "orc"
case s if s.startsWith("org.apache.spark.sql.hive.orc") => "orc"
+ case s if s.startsWith("org.apache.spark.sql.execution.datasources.orc")
=> "orc"
case s if s.equals("orcfile") => "orc"
case s if s.equals("parquetfile") => "parquet"
case s if s.equals("avrofile") => "avro"
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 688b619cd1bb5..5c9261c206ea7 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -159,10 +159,28 @@ class DataSourceWithHiveMetastoreCatalogSuite
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
)),
+ "org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat" ->
((
+ "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
+ "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
+ "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
+ )),
+
"orc" -> ((
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde"
+ )),
+
+ "org.apache.spark.sql.hive.orc" -> ((
+ "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+ "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+ "org.apache.hadoop.hive.ql.io.orc.OrcSerde"
+ )),
+
+ "org.apache.spark.sql.execution.datasources.orc.OrcFileFormat" -> ((
+ "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+ "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+ "org.apache.hadoop.hive.ql.io.orc.OrcSerde"
))
).foreach { case (provider, (inputFormat, outputFormat, serde)) =>
test(s"Persist non-partitioned $provider relation into metastore as
managed table") {
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index 7fefaf53939bd..c46512b6f5852 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -21,11 +21,9 @@ import java.io.File
import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.TestingUDT.{IntervalData, IntervalUDT}
-import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.datasources.orc.OrcSuite
import org.apache.spark.sql.hive.HiveUtils
import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.HiveSerDe
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
@@ -67,33 +65,6 @@ class HiveOrcSourceSuite extends OrcSuite with
TestHiveSingleton {
""".stripMargin)
}
- test("SPARK-22972: hive orc source") {
- val tableName = "normal_orc_as_source_hive"
- withTable(tableName) {
- sql(
- s"""
- |CREATE TABLE $tableName
- |USING org.apache.spark.sql.hive.orc
- |OPTIONS (
- | PATH '${new File(orcTableAsDir.getAbsolutePath).toURI}'
- |)
- """.stripMargin)
-
- val tableMetadata = spark.sessionState.catalog.getTableMetadata(
- TableIdentifier(tableName))
- assert(tableMetadata.storage.inputFormat ==
- Option("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
- assert(tableMetadata.storage.outputFormat ==
- Option("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
- assert(tableMetadata.storage.serde ==
- Option("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
- assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.hive.orc")
- .equals(HiveSerDe.sourceToSerDe("orc")))
- assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.orc")
- .equals(HiveSerDe.sourceToSerDe("orc")))
- }
- }
-
test("SPARK-19459/SPARK-18220: read char/varchar column written by Hive") {
val location = Utils.createTempDir()
val uri = location.toURI
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
