Weixin-Xu commented on code in PR #13492:
URL: https://github.com/apache/gluten/pull/13492#discussion_r4421508311
##########
backends-bolt/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala:
##########
@@ -335,24 +256,11 @@ case class ColumnarPartialGenerateExec(generateExec:
GenerateExec, child: SparkP
return Iterator.empty
}
- val colSizes = Array.fill(generateExec.generatorOutput.length)(0L)
- rowResults.foreach {
- row =>
- for (i <- variableWidthFields) {
- colSizes(i) = colSizes(i) + fieldsSizeGetter(i)(row, i)
- }
- }
-
- val rightResultVectors: Array[ArrowWritableColumnVector] =
- ArrowWritableColumnVector.allocateColumns(rowResults.length, colSizes,
rightSchema)
- val rightTargetRow = new ArrowColumnarRow(rightResultVectors)
-
- rowResults.foreach(row => writeRowUnsafe(row, rightTargetRow))
- rightTargetRow.finishWriteRow()
+ val rightTargetBatch = ArrowColumnVectors.fromRows(rightSchema,
rowResults.iterator)
Review Comment:
Generator termination is now driven by partition exhaustion rather than the
last nonempty batch. It runs exactly once and preserves the last input row
needed for termination output. Added a regression with a nonempty batch
followed by an empty batch to verify that buffered termination rows are emitted.
##########
backends-bolt/src/main/scala/org/apache/gluten/execution/ColumnarRangeExec.scala:
##########
@@ -98,17 +100,17 @@ case class ColumnarRangeExec(range: LogicalRange) extends
ColumnarRangeBaseExec
batchSize
)
- val vectors =
ArrowWritableColumnVector.allocateColumns(numRows, schema)
-
+ val root = ArrowColumnVectors.allocateRoot(schema)
+ val vector = root.getVector(0).asInstanceOf[BigIntVector]
+ vector.allocateNew(numRows)
for (i <- 0 until numRows) {
val value = current + i * step
Review Comment:
The remaining row count is now bounded in BigInt before conversion:
remaining.min(BigInt(batchSize)).max(BigInt(1)).toInt. Added regression
coverage for counts beyond the Int range to ensure they still produce normally
sized batches.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
