Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2762#discussion_r19258998
--- Diff:
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala ---
@@ -123,43 +93,88 @@ private[hive] trait HiveInspectors {
case pi: PrimitiveObjectInspector => pi.getPrimitiveJavaObject(data)
case li: ListObjectInspector =>
Option(li.getList(data))
- .map(_.map(unwrapData(_, li.getListElementObjectInspector)).toSeq)
+ .map(_.map(unwrap(_, li.getListElementObjectInspector)).toSeq)
.orNull
case mi: MapObjectInspector =>
Option(mi.getMap(data)).map(
_.map {
case (k,v) =>
- (unwrapData(k, mi.getMapKeyObjectInspector),
- unwrapData(v, mi.getMapValueObjectInspector))
+ (unwrap(k, mi.getMapKeyObjectInspector),
+ unwrap(v, mi.getMapValueObjectInspector))
}.toMap).orNull
case si: StructObjectInspector =>
val allRefs = si.getAllStructFieldRefs
new GenericRow(
allRefs.map(r =>
- unwrapData(si.getStructFieldData(data,r),
r.getFieldObjectInspector)).toArray)
+ unwrap(si.getStructFieldData(data,r),
r.getFieldObjectInspector)).toArray)
}
- /** Converts native catalyst types to the types expected by Hive */
- def wrap(a: Any): AnyRef = a match {
- case s: String => s: java.lang.String
- case i: Int => i: java.lang.Integer
- case b: Boolean => b: java.lang.Boolean
- case f: Float => f: java.lang.Float
- case d: Double => d: java.lang.Double
- case l: Long => l: java.lang.Long
- case l: Short => l: java.lang.Short
- case l: Byte => l: java.lang.Byte
- case b: BigDecimal => new HiveDecimal(b.underlying())
- case b: Array[Byte] => b
- case d: java.sql.Date => d
- case t: java.sql.Timestamp => t
- case s: Seq[_] => seqAsJavaList(s.map(wrap))
- case m: Map[_,_] =>
- // Some UDFs seem to assume we pass in a HashMap.
- val hashMap = new java.util.HashMap[AnyRef, AnyRef]()
- hashMap.putAll(m.map { case (k, v) => wrap(k) -> wrap(v) })
- hashMap
- case null => null
+ /**
+ * Converts native catalyst types to the types expected by Hive
+ * @param a the value to be wrapped
+ * @param oi wrapped object used by this ObjectInspector, which supposed
to be
+ * the ObjectInspector enumerated in functions:
+ * def toInspector(dataType: DataType)
+ * def toInspector(expr: Expression)
+ */
+ def wrap(a: Any, oi: ObjectInspector): AnyRef = if (a == null) {
+ null
+ } else {
+ oi match {
+ case x: ConstantObjectInspector => x.getWritableConstantValue
+ case x: PrimitiveObjectInspector => a match {
+ // TODO what if x.preferWritable() == true?
+ case s: String => s: java.lang.String
+ case i: Int => i: java.lang.Integer
+ case b: Boolean => b: java.lang.Boolean
+ case f: Float => f: java.lang.Float
+ case d: Double => d: java.lang.Double
+ case l: Long => l: java.lang.Long
+ case l: Short => l: java.lang.Short
+ case l: Byte => l: java.lang.Byte
+ case b: BigDecimal => new HiveDecimal(b.underlying())
+ case b: Array[Byte] => b
+ case d: java.sql.Date => d
+ case t: java.sql.Timestamp => t
+ }
+ case x: StructObjectInspector => {
+ val fieldRefs = x.getAllStructFieldRefs
+ val row = a.asInstanceOf[Seq[_]]
+ val result = new java.util.ArrayList[AnyRef](fieldRefs.length)
+ var i = 0
+ while(i < fieldRefs.length) {
+ result.add(wrap(row(i),
fieldRefs.get(i).getFieldObjectInspector))
+ i += 1
+ }
+
+ result
+ }
+ case x: ListObjectInspector =>
+ val list = new java.util.ArrayList[Object]
+ a.asInstanceOf[Seq[_]].foreach {
+ v => list.add(wrap(v, x.getListElementObjectInspector))
+ }
+ list
+ case x: MapObjectInspector =>
+ // Some UDFs seem to assume we pass in a HashMap.
+ val hashMap = new java.util.HashMap[AnyRef, AnyRef]()
+ hashMap.putAll(a.asInstanceOf[Map[_, _]].map {
+ case (k, v) =>
+ wrap(k, x.getMapKeyObjectInspector) -> wrap(v,
x.getMapValueObjectInspector)
+ })
+
+ hashMap
+ }
+ }
+
+ def wrap(row: Seq[Any], inspectors: Seq[ObjectInspector], cache:
Array[AnyRef])
+ : Array[AnyRef] = {
--- End diff --
```scala
def wrap(
row: Seq[Any],
inspectors: Seq[ObjectInspectlr],
cache: Array[AnyRef]): Array[AnyRef] = {
...
}
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
