This is an automated email from the ASF dual-hosted git repository.
beliefer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f9b9b64a11 [GLUTEN-11221][VL] Remove JArrayList, JHashMap, JMap from
VeloxIteratorApi (#11222)
f9b9b64a11 is described below
commit f9b9b64a11accf913e3249d4ac0454386c6260a4
Author: Jiaan Geng <[email protected]>
AuthorDate: Sat Nov 29 10:43:07 2025 +0800
[GLUTEN-11221][VL] Remove JArrayList, JHashMap, JMap from VeloxIteratorApi
(#11222)
---
.../backendsapi/velox/VeloxIteratorApi.scala | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
index 8c496b7419..c6ad7d5029 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
@@ -43,9 +43,10 @@ import org.apache.spark.util.SparkDirectoryUtil
import java.lang.{Long => JLong}
import java.nio.charset.StandardCharsets
import java.time.ZoneOffset
-import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Map => JMap,
UUID}
+import java.util.UUID
import scala.collection.JavaConverters._
+import scala.collection.mutable
class VeloxIteratorApi extends IteratorApi with Logging {
@@ -104,7 +105,7 @@ class VeloxIteratorApi extends IteratorApi with Logging {
lengths.asJava,
fileSizes.asJava,
modificationTimes.asJava,
- partitionColumns.asJava,
+ partitionColumns.map(_.asJava).asJava,
metadataColumns.asJava,
fileFormat,
locations.toList.asJava,
@@ -145,10 +146,10 @@ class VeloxIteratorApi extends IteratorApi with Logging {
private def getPartitionColumns(
schema: StructType,
- partitionedFiles: Seq[PartitionedFile]): Seq[JMap[String, String]] = {
+ partitionedFiles: Seq[PartitionedFile]): Seq[Map[String, String]] = {
partitionedFiles.map {
partitionedFile =>
- val partitionColumn = new JHashMap[String, String]()
+ val partitionColumn = mutable.Map[String, String]()
for (i <- 0 until partitionedFile.partitionValues.numFields) {
val partitionColumnValue = if
(partitionedFile.partitionValues.isNullAt(i)) {
ExternalCatalogUtils.DEFAULT_PARTITION_NAME
@@ -168,9 +169,9 @@ class VeloxIteratorApi extends IteratorApi with Logging {
case _ => pv.toString
}
}
- partitionColumn.put(schema.names(i), partitionColumnValue)
+ partitionColumn += (schema.names(i) -> partitionColumnValue)
}
- partitionColumn
+ partitionColumn.toMap
}
}
@@ -192,10 +193,9 @@ class VeloxIteratorApi extends IteratorApi with Logging {
inputPartition.isInstanceOf[GlutenPartition],
"Velox backend only accept GlutenPartition.")
- val columnarNativeIterators =
- new JArrayList[ColumnarBatchInIterator](inputIterators.map {
- iter => new ColumnarBatchInIterator(BackendsApiManager.getBackendName,
iter.asJava)
- }.asJava)
+ val columnarNativeIterators = inputIterators.map {
+ iter => new ColumnarBatchInIterator(BackendsApiManager.getBackendName,
iter.asJava)
+ }
val extraConf = Map(GlutenConfig.COLUMNAR_CUDF_ENABLED.key ->
enableCudf.toString).asJava
val transKernel =
NativePlanEvaluator.create(BackendsApiManager.getBackendName, extraConf)
@@ -214,7 +214,7 @@ class VeloxIteratorApi extends IteratorApi with Logging {
transKernel.createKernelWithBatchIterator(
inputPartition.plan,
splitInfoByteArray,
- columnarNativeIterators,
+ columnarNativeIterators.asJava,
partitionIndex,
BackendsApiManager.getSparkPlanExecApiInstance.rewriteSpillPath(spillDirPath)
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
