JNSimba commented on code in PR #151:
URL:
https://github.com/apache/doris-spark-connector/pull/151#discussion_r1378573326
##########
spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala:
##########
@@ -147,72 +151,63 @@ private[spark] object SchemaUtils {
def rowColumnValue(row: SpecializedGetters, ordinal: Int, dataType:
DataType): Any = {
- dataType match {
- case NullType => DataUtil.NULL_VALUE
- case BooleanType => row.getBoolean(ordinal)
- case ByteType => row.getByte(ordinal)
- case ShortType => row.getShort(ordinal)
- case IntegerType => row.getInt(ordinal)
- case LongType => row.getLong(ordinal)
- case FloatType => row.getFloat(ordinal)
- case DoubleType => row.getDouble(ordinal)
- case StringType =>
Option(row.getUTF8String(ordinal)).map(_.toString).getOrElse(DataUtil.NULL_VALUE)
- case TimestampType =>
- LocalDateTime.ofEpochSecond(row.getLong(ordinal) / 100000,
(row.getLong(ordinal) % 1000).toInt, ZoneOffset.UTC)
- new Timestamp(row.getLong(ordinal) / 1000).toString
- case DateType => DateTimeUtils.toJavaDate(row.getInt(ordinal)).toString
- case BinaryType => row.getBinary(ordinal)
- case dt: DecimalType => row.getDecimal(ordinal, dt.precision, dt.scale)
- case at: ArrayType =>
- val arrayData = row.getArray(ordinal)
- if (arrayData == null) DataUtil.NULL_VALUE
- else if(arrayData.numElements() == 0) "[]"
- else {
- (0 until arrayData.numElements()).map(i => {
- if (arrayData.isNullAt(i)) null else rowColumnValue(arrayData, i,
at.elementType)
- }).mkString("[", ",", "]")
- }
-
- case mt: MapType =>
- val mapData = row.getMap(ordinal)
- val keys = mapData.keyArray()
- val values = mapData.valueArray()
- val sb = StringBuilder.newBuilder
- sb.append("{")
- var i = 0
- while (i < keys.numElements()) {
- rowColumnValue(keys, i, mt.keyType) -> rowColumnValue(values, i,
mt.valueType)
- sb.append(quoteData(rowColumnValue(keys, i, mt.keyType), mt.keyType))
- .append(":").append(quoteData(rowColumnValue(values, i,
mt.valueType), mt.valueType))
- .append(",")
- i += 1
- }
- if (i > 0) sb.dropRight(1)
- sb.append("}").toString
- case st: StructType =>
- val structData = row.getStruct(ordinal, st.length)
- val sb = StringBuilder.newBuilder
- sb.append("{")
- var i = 0
- while (i < structData.numFields) {
- val field = st.get(i)
- sb.append(s""""${field.name}":""")
- .append(quoteData(rowColumnValue(structData, i, field.dataType),
field.dataType))
- .append(",")
- i += 1
- }
- if (i > 0) sb.dropRight(1)
- sb.append("}").toString
- case _ => throw new DorisException(s"Unsupported spark type:
${dataType.typeName}")
+ if (row.isNullAt(ordinal)) null
+ else {
+ dataType match {
+ case NullType => DataUtil.NULL_VALUE
+ case BooleanType => row.getBoolean(ordinal)
+ case ByteType => row.getByte(ordinal)
+ case ShortType => row.getShort(ordinal)
+ case IntegerType => row.getInt(ordinal)
+ case LongType => row.getLong(ordinal)
+ case FloatType => row.getFloat(ordinal)
+ case DoubleType => row.getDouble(ordinal)
+ case StringType =>
Option(row.getUTF8String(ordinal)).map(_.toString).getOrElse(DataUtil.NULL_VALUE)
+ case TimestampType =>
+ LocalDateTime.ofEpochSecond(row.getLong(ordinal) / 100000,
(row.getLong(ordinal) % 1000).toInt, ZoneOffset.UTC)
+ new Timestamp(row.getLong(ordinal) / 1000).toString
+ case DateType => DateTimeUtils.toJavaDate(row.getInt(ordinal)).toString
+ case BinaryType => row.getBinary(ordinal)
+ case dt: DecimalType => row.getDecimal(ordinal, dt.precision, dt.scale)
+ case at: ArrayType =>
+ val arrayData = row.getArray(ordinal)
+ if (arrayData == null) DataUtil.NULL_VALUE
+ else {
+ (0 until arrayData.numElements()).map(i => {
+ if (arrayData.isNullAt(i)) null else rowColumnValue(arrayData,
i, at.elementType)
+ }).mkString("[", ",", "]")
+ }
+ case mt: MapType =>
+ val mapData = row.getMap(ordinal)
+ if (mapData == null) "{}"
Review Comment:
Should null not be processed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
