This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d6fe65d161 [GLUTEN-9475][VL][FOLLOWUP] Avoid overflow in calculation
of numRows (#9704)
d6fe65d161 is described below
commit d6fe65d16187f036ff190931872b054be2725416
Author: Zhen Wang <[email protected]>
AuthorDate: Wed May 21 10:02:49 2025 +0800
[GLUTEN-9475][VL][FOLLOWUP] Avoid overflow in calculation of numRows (#9704)
---
.../scala/org/apache/spark/sql/execution/BroadcastUtils.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
index 10e72ab702..71080877ed 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.vectorized.ColumnarBatch
import org.apache.spark.task.TaskResources
-import scala.collection.mutable.ArrayBuffer;
+import scala.collection.mutable.ArrayBuffer
// Utility methods to convert Vanilla broadcast relations from/to Velox
broadcast relations.
// FIXME: Truncate output with batch size.
@@ -168,12 +168,12 @@ object BroadcastUtils {
ColumnarBatches.retain(b)
b
})
- var rowNums = 0
+ var numRows: Long = 0
val values = filtered
.map(
b => {
val handle =
ColumnarBatches.getNativeHandle(BackendsApiManager.getBackendName, b)
- rowNums += b.numRows()
+ numRows += b.numRows()
try {
ColumnarBatchSerializerJniWrapper
.create(
@@ -188,7 +188,7 @@ object BroadcastUtils {
})
.toArray
if (values.nonEmpty) {
- new ColumnarBatchSerializeResult(rowNums, values)
+ new ColumnarBatchSerializeResult(numRows, values)
} else {
ColumnarBatchSerializeResult.EMPTY
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
