SaurabhChawla100 commented on a change in pull request #29045:
URL: https://github.com/apache/spark/pull/29045#discussion_r453278778
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
##########
@@ -181,10 +183,19 @@ class OrcFileFormat
val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
val requestedColIdsOrEmptyFile =
Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) {
reader =>
+ // for ORC file written by Hive, no field names
+ // in the physical schema, there is a need to send the
+ // entire dataSchema instead of required schema
+ val orcFieldNames = reader.getSchema.getFieldNames.asScala
+ if (orcFieldNames.forall(_.startsWith("_col"))) {
+ resultSchemaString =
OrcUtils.orcTypeDescriptionString(actualSchema)
+ }
Review comment:
I have refactored the code an include this in OrcUtils.requestedColumnIds
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
##########
@@ -77,4 +77,44 @@ class OrcColumnarBatchReaderSuite extends QueryTest with
SharedSparkSession {
assert(p1.getUTF8String(0) === partitionValues.getUTF8String(0))
}
}
+
+ test("SPARK-32234: orc data created by the hive tables having _col fields
name") {
+ withTable("test_date_hive_orc") {
+ spark.sql(
+ """
+ | CREATE TABLE test_date_hive_orc
+ | (_col1 INT, _col2 STRING, _col3 INT)
+ | USING orc
+ """.stripMargin)
+ spark.sql(
+ """
+ | INSERT INTO
+ | test_date_hive_orc
+ | VALUES(9, '12', 2020)
+ """.stripMargin)
+
+ val df = spark.sql("SELECT _col2 from test_date_hive_orc")
+ checkAnswer(df, Row("12"))
+ }
+ }
+
+ test("SPARK-32234: orc data created by the spark having proper fields name")
{
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
