abellina commented on a change in pull request #24795: [SPARK-27945][SQL]
Minimal changes to support columnar processing
URL: https://github.com/apache/spark/pull/24795#discussion_r293935318
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
##########
@@ -1270,6 +1273,211 @@ class ColumnarBatchSuite extends SparkFunSuite {
allocator.close()
}
+ test("RowToColumnarExec") {
+ val schema = StructType(
+ StructField("str", StringType) ::
+ StructField("bool", BooleanType) ::
+ StructField("byte", ByteType) ::
+ StructField("short", ShortType) ::
+ StructField("int", IntegerType) ::
+ StructField("long", LongType) ::
+ StructField("float", FloatType) ::
+ StructField("double", DoubleType) ::
+ StructField("decimal", DecimalType(25, 5)) ::
+ StructField("date", DateType) ::
+ StructField("ts", TimestampType) ::
+ StructField("cal", CalendarIntervalType) ::
+ StructField("arr_of_int", ArrayType(IntegerType)) ::
+ StructField("int_and_int", StructType(
+ StructField("int1", IntegerType) ::
+ StructField("int2", IntegerType) ::
+ Nil
+ )) ::
+ StructField("int_to_int", MapType(IntegerType, IntegerType)) ::
+ Nil)
+ var mapBuilder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
+ mapBuilder.put(1, 10)
+ mapBuilder.put(20, null)
+ val row1 = new GenericInternalRow(Array[Any](
+ UTF8String.fromString("a string"),
+ true,
+ 1.toByte,
+ 2.toShort,
+ 3,
+ Long.MaxValue,
+ 0.25.toFloat,
+ 0.75D,
+ Decimal("1234.23456"),
+ DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("2015-01-01")),
+ DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("2015-01-01
23:50:59.123")),
+ new CalendarInterval(1, 0),
+ new GenericArrayData(Array(1, 2, 3, 4, null)),
+ new GenericInternalRow(Array[Any](5.asInstanceOf[Any], 10)),
+ mapBuilder.build()
+ ))
+
+ mapBuilder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
+ mapBuilder.put(30, null)
+ mapBuilder.put(40, 50)
+ val row2 = new GenericInternalRow(Array[Any](
+ UTF8String.fromString("second string"),
+ false,
+ -1.toByte,
+ 17.toShort,
+ Int.MinValue,
+ 987654321L,
+ Float.NaN,
+ Double.PositiveInfinity,
+ Decimal("0.01000"),
+ DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("1875-12-12")),
+ DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("1880-01-05
12:45:21.321")),
+ new CalendarInterval(-10, -100),
+ new GenericArrayData(Array(5, 10, -100)),
+ new GenericInternalRow(Array[Any](20.asInstanceOf[Any], null)),
+ mapBuilder.build()
+ ))
+
+ val row3 = new GenericInternalRow(Array[Any](
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ ))
+
+ val converter = new RowToColumnConverter(schema)
+ val columns = OnHeapColumnVector.allocateColumns(3, schema)
+ val batch = new ColumnarBatch(columns.toArray, 3)
+ try {
+ converter.convert(row1, columns.toArray)
+ converter.convert(row2, columns.toArray)
+ converter.convert(row3, columns.toArray)
+
+ assert(columns(0).dataType() == StringType)
+ assert(columns(0).getUTF8String(0).toString == "a string")
+ assert(columns(0).getUTF8String(1).toString == "second string")
+ assert(columns(0).isNullAt(2))
+
+ assert(columns(1).dataType() == BooleanType)
+ assert(columns(1).getBoolean(0) == true)
+ assert(columns(1).getBoolean(1) == false)
+ assert(columns(1).isNullAt(2))
+
+ assert(columns(2).dataType() == ByteType)
+ assert(columns(2).getByte(0) == 1.toByte)
+ assert(columns(2).getByte(1) == -1.toByte)
+ assert(columns(2).isNullAt(2))
+
+ assert(columns(3).dataType() == ShortType)
+ assert(columns(3).getShort(0) == 2.toByte)
Review comment:
nit, toShort
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
