MaxGekk commented on code in PR #40632:
URL: https://github.com/apache/spark/pull/40632#discussion_r1199718277
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala:
##########
@@ -119,8 +119,9 @@ class JacksonParser(
} else {
new NoopFilters
}
- (parser: JsonParser) => parseJsonToken[Iterable[InternalRow]](parser, st) {
- case START_OBJECT => convertObject(parser, st, fieldConverters,
jsonFilters, isRoot = true)
+ (parser: JsonParser) =>
+ parseJsonToken[Iterable[InternalRow]](parser, st) {
+ case START_OBJECT => convertObject(parser, st, fieldConverters,
jsonFilters, isRoot = true)
Review Comment:
Let avoid unnecessary changes, or there are any reasons for this?
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala:
##########
@@ -134,54 +135,58 @@ class JacksonParser(
// List([str_a_1,null])
// List([str_a_2,null], [null,str_b_3])
//
- case START_ARRAY if allowArrayAsStructs =>
- val array = convertArray(parser, elementConverter, isRoot = true)
- // Here, as we support reading top level JSON arrays and take every
element
- // in such an array as a row, this case is possible.
- if (array.numElements() == 0) {
- Array.empty[InternalRow]
- } else {
- array.toArray[InternalRow](schema)
- }
- case START_ARRAY =>
- throw QueryExecutionErrors.cannotParseJsonArraysAsStructsError()
- }
+ case START_ARRAY if allowArrayAsStructs =>
+ val array = convertArray(parser, elementConverter, isRoot = true)
+ // Here, as we support reading top level JSON arrays and take every
element
+ // in such an array as a row, this case is possible.
+ if (array.numElements() == 0) {
+ Array.empty[InternalRow]
+ } else {
+ array.toArray[InternalRow](schema)
+ }
+ case START_ARRAY =>
+ throw QueryExecutionErrors.cannotParseJsonArraysAsStructsError("")
+ }
Review Comment:
Have you changes only indentations? If so, please, revert this back.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala:
##########
@@ -134,54 +135,58 @@ class JacksonParser(
// List([str_a_1,null])
// List([str_a_2,null], [null,str_b_3])
//
- case START_ARRAY if allowArrayAsStructs =>
- val array = convertArray(parser, elementConverter, isRoot = true)
- // Here, as we support reading top level JSON arrays and take every
element
- // in such an array as a row, this case is possible.
- if (array.numElements() == 0) {
- Array.empty[InternalRow]
- } else {
- array.toArray[InternalRow](schema)
- }
- case START_ARRAY =>
- throw QueryExecutionErrors.cannotParseJsonArraysAsStructsError()
- }
+ case START_ARRAY if allowArrayAsStructs =>
+ val array = convertArray(parser, elementConverter, isRoot = true)
+ // Here, as we support reading top level JSON arrays and take every
element
+ // in such an array as a row, this case is possible.
+ if (array.numElements() == 0) {
+ Array.empty[InternalRow]
+ } else {
+ array.toArray[InternalRow](schema)
+ }
+ case START_ARRAY =>
+ throw QueryExecutionErrors.cannotParseJsonArraysAsStructsError("")
+ }
}
private def makeMapRootConverter(mt: MapType): JsonParser =>
Iterable[InternalRow] = {
val fieldConverter = makeConverter(mt.valueType)
- (parser: JsonParser) => parseJsonToken[Iterable[InternalRow]](parser, mt) {
- case START_OBJECT => Some(InternalRow(convertMap(parser,
fieldConverter)))
- }
+ (parser: JsonParser) =>
+ parseJsonToken[Iterable[InternalRow]](parser, mt) {
+ case START_OBJECT => Some(InternalRow(convertMap(parser,
fieldConverter)))
+ }
}
private def makeArrayRootConverter(at: ArrayType): JsonParser =>
Iterable[InternalRow] = {
val elemConverter = makeConverter(at.elementType)
- (parser: JsonParser) => parseJsonToken[Iterable[InternalRow]](parser, at) {
- case START_ARRAY => Some(InternalRow(convertArray(parser,
elemConverter)))
- case START_OBJECT if at.elementType.isInstanceOf[StructType] =>
- // This handles the case when an input JSON object is a structure but
- // the specified schema is an array of structures. In that case, the
input JSON is
- // considered as an array of only one element of struct type.
- // This behavior was introduced by changes for SPARK-19595.
- //
- // For example, if the specified schema is ArrayType(new
StructType().add("i", IntegerType))
- // and JSON input as below:
- //
- // [{"i": 1}, {"i": 2}]
- // [{"i": 3}]
- // {"i": 4}
- //
- // The last row is considered as an array with one element, and result
of conversion:
- //
- // Seq(Row(1), Row(2))
- // Seq(Row(3))
- // Seq(Row(4))
- //
- val st = at.elementType.asInstanceOf[StructType]
- val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
- Some(InternalRow(new GenericArrayData(convertObject(parser, st,
fieldConverters).toArray)))
- }
+ (parser: JsonParser) =>
+ parseJsonToken[Iterable[InternalRow]](parser, at) {
+ case START_ARRAY => Some(InternalRow(convertArray(parser,
elemConverter)))
+ case START_OBJECT if at.elementType.isInstanceOf[StructType] =>
+ // This handles the case when an input JSON object is a structure but
+ // the specified schema is an array of structures. In that case, the
input JSON is
+ // considered as an array of only one element of struct type.
+ // This behavior was introduced by changes for SPARK-19595.
+ //
+ // For example, if the specified schema is
+ // ArrayType(new StructType().add("i", IntegerType))
+ // and JSON input as below:
+ //
+ // [{"i": 1}, {"i": 2}]
+ // [{"i": 3}]
+ // {"i": 4}
+ //
+ // The last row is considered as an array with one element, and
result of conversion:
+ //
+ // Seq(Row(1), Row(2))
+ // Seq(Row(3))
+ // Seq(Row(4))
+ //
+ val st = at.elementType.asInstanceOf[StructType]
+ val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
+ Some(InternalRow(new GenericArrayData(convertObject(parser, st,
fieldConverters)
+ .toArray)))
+ }
Review Comment:
Please, highlight what you changed here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
