Github user dafrista commented on a diff in the pull request:
https://github.com/apache/spark/pull/13676#discussion_r67092384
--- Diff:
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala ---
@@ -243,137 +243,12 @@ private[hive] trait HiveInspectors {
* @param data the data in Hive type
* @param oi the ObjectInspector associated with the Hive Type
* @return convert the data into catalyst type
- * TODO return the function of (data => Any) instead for performance
consideration
*
- * Strictly follows the following order in unwrapping (constant OI has
the higher priority):
- * Constant Null object inspector =>
- * return null
- * Constant object inspector =>
- * extract the value from constant object inspector
- * Check whether the `data` is null =>
- * return null if true
- * If object inspector prefers writable =>
- * extract writable from `data` and then get the catalyst type from
the writable
- * Extract the java object directly from the object inspector
- *
- * NOTICE: the complex data type requires recursive unwrapping.
+ * Use unwrapperFor's (data => Any) instead for performance
consideration.
*/
- def unwrap(data: Any, oi: ObjectInspector): Any = oi match {
- case coi: ConstantObjectInspector if coi.getWritableConstantValue ==
null => null
- case poi: WritableConstantStringObjectInspector =>
- UTF8String.fromString(poi.getWritableConstantValue.toString)
- case poi: WritableConstantHiveVarcharObjectInspector =>
-
UTF8String.fromString(poi.getWritableConstantValue.getHiveVarchar.getValue)
- case poi: WritableConstantHiveCharObjectInspector =>
-
UTF8String.fromString(poi.getWritableConstantValue.getHiveChar.getValue)
- case poi: WritableConstantHiveDecimalObjectInspector =>
- HiveShim.toCatalystDecimal(
- PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector,
- poi.getWritableConstantValue.getHiveDecimal)
- case poi: WritableConstantTimestampObjectInspector =>
- val t = poi.getWritableConstantValue
- t.getSeconds * 1000000L + t.getNanos / 1000L
- case poi: WritableConstantIntObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantDoubleObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantBooleanObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantLongObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantFloatObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantShortObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantByteObjectInspector =>
- poi.getWritableConstantValue.get()
- case poi: WritableConstantBinaryObjectInspector =>
- val writable = poi.getWritableConstantValue
- val temp = new Array[Byte](writable.getLength)
- System.arraycopy(writable.getBytes, 0, temp, 0, temp.length)
- temp
- case poi: WritableConstantDateObjectInspector =>
- DateTimeUtils.fromJavaDate(poi.getWritableConstantValue.get())
- case mi: StandardConstantMapObjectInspector =>
- // take the value from the map inspector object, rather than the
input data
- val keyValues = mi.getWritableConstantValue.asScala.toSeq
- val keys = keyValues.map(kv => unwrap(kv._1,
mi.getMapKeyObjectInspector)).toArray
- val values = keyValues.map(kv => unwrap(kv._2,
mi.getMapValueObjectInspector)).toArray
- ArrayBasedMapData(keys, values)
- case li: StandardConstantListObjectInspector =>
- // take the value from the list inspector object, rather than the
input data
- val values = li.getWritableConstantValue.asScala
- .map(unwrap(_, li.getListElementObjectInspector))
- .toArray
- new GenericArrayData(values)
- // if the value is null, we don't care about the object inspector type
- case _ if data == null => null
- case poi: VoidObjectInspector => null // always be null for void
object inspector
- case pi: PrimitiveObjectInspector => pi match {
- // We think HiveVarchar/HiveChar is also a String
- case hvoi: HiveVarcharObjectInspector if hvoi.preferWritable() =>
-
UTF8String.fromString(hvoi.getPrimitiveWritableObject(data).getHiveVarchar.getValue)
- case hvoi: HiveVarcharObjectInspector =>
- UTF8String.fromString(hvoi.getPrimitiveJavaObject(data).getValue)
- case hvoi: HiveCharObjectInspector if hvoi.preferWritable() =>
-
UTF8String.fromString(hvoi.getPrimitiveWritableObject(data).getHiveChar.getValue)
- case hvoi: HiveCharObjectInspector =>
- UTF8String.fromString(hvoi.getPrimitiveJavaObject(data).getValue)
- case x: StringObjectInspector if x.preferWritable() =>
- // Text is in UTF-8 already. No need to convert again via
fromString. Copy bytes
- val wObj = x.getPrimitiveWritableObject(data)
- val result = wObj.copyBytes()
- UTF8String.fromBytes(result, 0, result.length)
- case x: StringObjectInspector =>
- UTF8String.fromString(x.getPrimitiveJavaObject(data))
- case x: IntObjectInspector if x.preferWritable() => x.get(data)
- case x: BooleanObjectInspector if x.preferWritable() => x.get(data)
- case x: FloatObjectInspector if x.preferWritable() => x.get(data)
- case x: DoubleObjectInspector if x.preferWritable() => x.get(data)
- case x: LongObjectInspector if x.preferWritable() => x.get(data)
- case x: ShortObjectInspector if x.preferWritable() => x.get(data)
- case x: ByteObjectInspector if x.preferWritable() => x.get(data)
- case x: HiveDecimalObjectInspector => HiveShim.toCatalystDecimal(x,
data)
- case x: BinaryObjectInspector if x.preferWritable() =>
- // BytesWritable.copyBytes() only available since Hadoop2
- // In order to keep backward-compatible, we have to copy the
- // bytes with old apis
- val bw = x.getPrimitiveWritableObject(data)
- val result = new Array[Byte](bw.getLength())
- System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength())
- result
- case x: DateObjectInspector if x.preferWritable() =>
-
DateTimeUtils.fromJavaDate(x.getPrimitiveWritableObject(data).get())
- case x: DateObjectInspector =>
DateTimeUtils.fromJavaDate(x.getPrimitiveJavaObject(data))
- case x: TimestampObjectInspector if x.preferWritable() =>
- val t = x.getPrimitiveWritableObject(data)
- t.getSeconds * 1000000L + t.getNanos / 1000L
- case ti: TimestampObjectInspector =>
- DateTimeUtils.fromJavaTimestamp(ti.getPrimitiveJavaObject(data))
- case _ => pi.getPrimitiveJavaObject(data)
- }
- case li: ListObjectInspector =>
- Option(li.getList(data))
- .map { l =>
- val values = l.asScala.map(unwrap(_,
li.getListElementObjectInspector)).toArray
- new GenericArrayData(values)
- }
- .orNull
- case mi: MapObjectInspector =>
- val map = mi.getMap(data)
- if (map == null) {
- null
- } else {
- val keyValues = map.asScala.toSeq
- val keys = keyValues.map(kv => unwrap(kv._1,
mi.getMapKeyObjectInspector)).toArray
- val values = keyValues.map(kv => unwrap(kv._2,
mi.getMapValueObjectInspector)).toArray
- ArrayBasedMapData(keys, values)
- }
- // currently, hive doesn't provide the ConstantStructObjectInspector
- case si: StructObjectInspector =>
- val allRefs = si.getAllStructFieldRefs
- InternalRow.fromSeq(allRefs.asScala.map(
- r => unwrap(si.getStructFieldData(data, r),
r.getFieldObjectInspector)))
+ def unwrap(data: Any, oi: ObjectInspector): Any = {
+ val unwrapper = unwrapperFor(oi)
+ unwrapper(data)
--- End diff --
There is no improvement when calling unwrap directly. This change moves the
unwrap logic to unwrapperFor. This improves performance for ORC files, because
`OrcFileFormat` instead calls unwrapperFor to return a function for each field,
then unwraps each row with this function (see
https://github.com/apache/spark/blob/master/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala#L356).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
