Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/18875#discussion_r137227541
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
---
@@ -22,24 +22,44 @@ import java.io.Writer
import com.fasterxml.jackson.core._
import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils,
MapData}
import org.apache.spark.sql.types._
private[sql] class JacksonGenerator(
- schema: StructType,
+ rowSchema: DataType,
writer: Writer,
options: JSONOptions) {
+
// A `ValueWriter` is responsible for writing a field of an
`InternalRow` to appropriate
// JSON data. Here we are using `SpecializedGetters` rather than
`InternalRow` so that
// we can directly access data in `ArrayData` without the help of
`SpecificMutableRow`.
private type ValueWriter = (SpecializedGetters, Int) => Unit
+ // `JackGenerator` only supports to write out a struct, an array of
struct or an arbitrary map
+ rowSchema match {
+ case _: StructType | _: MapType =>
+ TypeCheckResult.TypeCheckSuccess
+ case _ => TypeCheckResult.TypeCheckFailure(
+ s"Input type ${rowSchema.simpleString} must be a struct or a map")
+ }
+
// `ValueWriter`s for all fields of the schema
- private val rootFieldWriters: Array[ValueWriter] =
schema.map(_.dataType).map(makeWriter).toArray
+ private lazy val rootFieldWriters: Array[ValueWriter] = {
+
rowSchema.asInstanceOf[StructType].map(_.dataType).map(makeWriter).toArray
--- End diff --
We should take care when accessing `rootFieldWriters` if the given data
type is a map type. E.g.:
private lazy val rootFieldWriters: Array[ValueWriter] = dataType match {
case st: StructType => st.map(_.dataType).map(makeWriter).toArray
case mt: MapType => throw UnsupportedOperationException(".......")
}
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
