maropu commented on a change in pull request #29654:
URL: https://github.com/apache/spark/pull/29654#discussion_r485262885
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
##########
@@ -182,29 +181,29 @@ private[columnar] case object RunLengthEncoding extends
CompressionScheme {
private var _uncompressedSize = 0
private var _compressedSize = 0
- // Using `MutableRow` to store the last value to avoid boxing/unboxing
cost.
- private val lastValue = new SpecificInternalRow(Seq(columnType.dataType))
+ private var lastValue: T#InternalType = _
private var lastRun = 0
override def uncompressedSize: Int = _uncompressedSize
override def compressedSize: Int = _compressedSize
+
override def gatherCompressibilityStats(row: InternalRow, ordinal: Int):
Unit = {
val value = columnType.getField(row, ordinal)
val actualSize = columnType.actualSize(row, ordinal)
_uncompressedSize += actualSize
- if (lastValue.isNullAt(0)) {
- columnType.copyField(row, ordinal, lastValue, 0)
+ if (lastValue == null) {
+ lastValue = columnType.clone(columnType.getField(row, ordinal))
lastRun = 1
_compressedSize += actualSize + 4
} else {
- if (columnType.getField(lastValue, 0) == value) {
+ if (lastValue == value) {
Review comment:
Could we get actual performance differences by this change? Did you run
microbenchmarks to check them?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
