Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/19470#discussion_r144457479
--- Diff:
sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala ---
@@ -272,25 +272,35 @@ private[orc] object OrcRelation extends
HiveInspectors {
def unwrapOrcStructs(
conf: Configuration,
dataSchema: StructType,
+ requiredSchema: StructType,
maybeStructOI: Option[StructObjectInspector],
iterator: Iterator[Writable]): Iterator[InternalRow] = {
val deserializer = new OrcSerde
- val mutableRow = new SpecificInternalRow(dataSchema.map(_.dataType))
- val unsafeProjection = UnsafeProjection.create(dataSchema)
+ val mutableRow = new
SpecificInternalRow(requiredSchema.map(_.dataType))
+ val unsafeProjection = UnsafeProjection.create(requiredSchema)
def unwrap(oi: StructObjectInspector): Iterator[InternalRow] = {
- val (fieldRefs, fieldOrdinals) = dataSchema.zipWithIndex.map {
- case (field, ordinal) => oi.getStructFieldRef(field.name) ->
ordinal
+ val (fieldRefs, fieldOrdinals) = requiredSchema.zipWithIndex.map {
+ case (field, ordinal) =>
+ var ref = oi.getStructFieldRef(field.name)
+ if (ref == null) {
+ val maybeIndex = dataSchema.getFieldIndex(field.name)
--- End diff --
the `requiredSchema` is guaranteed to be contained in the `dataSchema`.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
