wchevreuil commented on code in PR #169:
URL: https://github.com/apache/hbase-connectors/pull/169#discussion_r4231431497
##########
spark4/hbase-spark4/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseDataWriter.scala:
##########
@@ -128,21 +128,27 @@ class HBaseDataWriter(
private val MILLIS_PER_DAY = 86400000L
private def getValueBytes(row: InternalRow, idx: Int, field: Field):
Array[Byte] = {
- field.dt match {
- case BooleanType => Bytes.toBytes(row.getBoolean(idx))
- case ByteType => Array(row.getByte(idx))
- case ShortType => Bytes.toBytes(row.getShort(idx))
- case IntegerType => Bytes.toBytes(row.getInt(idx))
- case LongType => Bytes.toBytes(row.getLong(idx))
- case FloatType => Bytes.toBytes(row.getFloat(idx))
- case DoubleType => Bytes.toBytes(row.getDouble(idx))
- case StringType => Bytes.toBytes(row.getUTF8String(idx).toString)
- case BinaryType => row.getBinary(idx)
- case DateType => Bytes.toBytes(row.getInt(idx).toLong * MILLIS_PER_DAY)
- case TimestampType => Bytes.toBytes(row.getLong(idx) / 1000)
- case dt: DecimalType =>
- Bytes.toBytes(row.getDecimal(idx, dt.precision,
dt.scale).toJavaBigDecimal)
- case _ => throw new UnsupportedOperationException(s"unsupported data
type ${field.dt}")
+ if (field.exeSchema.isDefined) {
+ val struct = row.getStruct(idx, field.dt.asInstanceOf[StructType].length)
+ val record = field.catalystToAvroFromInternalRow(struct)
+ AvroSerdes.serialize(record, field.exeSchema.get)
Review Comment:
Addressing on the next commit. Fix is to check on field.dt first and use
row.get(idx, field.dt) for primitives, or getStruct for StructType.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
