This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 053b7ccc minor: move shuffle classes from common to spark (#1193)
053b7ccc is described below
commit 053b7ccc18c8397c7035a79b4df092beddf28289
Author: Andy Grove <[email protected]>
AuthorDate: Sat Dec 21 17:59:34 2024 -0700
minor: move shuffle classes from common to spark (#1193)
---
.../comet/execution/shuffle/ArrowReaderIterator.scala | 0
.../shuffle/CometBlockStoreShuffleReader.scala | 17 ++++-------------
.../execution/shuffle/CometShuffleDependency.scala | 0
.../comet/execution/shuffle/CometShuffledRowRDD.scala | 4 ++--
.../execution/shuffle/IpcInputStreamIterator.scala | 9 +++------
.../sql/comet/execution/shuffle/ShuffleUtils.scala | 0
6 files changed, 9 insertions(+), 21 deletions(-)
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ArrowReaderIterator.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ArrowReaderIterator.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ArrowReaderIterator.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ArrowReaderIterator.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala
similarity index 92%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala
index e026cbeb..74c65595 100644
---
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala
+++
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala
@@ -21,21 +21,12 @@ package org.apache.spark.sql.comet.execution.shuffle
import java.io.InputStream
-import org.apache.spark.InterruptibleIterator
-import org.apache.spark.MapOutputTracker
-import org.apache.spark.SparkEnv
-import org.apache.spark.TaskContext
-import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config
+import org.apache.spark.{InterruptibleIterator, MapOutputTracker, SparkEnv,
TaskContext}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.SerializerManager
-import org.apache.spark.shuffle.BaseShuffleHandle
-import org.apache.spark.shuffle.ShuffleReader
-import org.apache.spark.shuffle.ShuffleReadMetricsReporter
-import org.apache.spark.storage.BlockId
-import org.apache.spark.storage.BlockManager
-import org.apache.spark.storage.BlockManagerId
-import org.apache.spark.storage.ShuffleBlockFetcherIterator
+import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleReader,
ShuffleReadMetricsReporter}
+import org.apache.spark.storage.{BlockId, BlockManager, BlockManagerId,
ShuffleBlockFetcherIterator}
import org.apache.spark.util.CompletionIterator
/**
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffledRowRDD.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffledRowRDD.scala
similarity index 95%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffledRowRDD.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffledRowRDD.scala
index af78ed29..ba6fc588 100644
---
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffledRowRDD.scala
+++
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffledRowRDD.scala
@@ -19,10 +19,10 @@
package org.apache.spark.sql.comet.execution.shuffle
-import org.apache.spark.{Dependency, MapOutputTrackerMaster, Partition,
Partitioner, ShuffleDependency, SparkEnv, TaskContext}
+import org.apache.spark._
import org.apache.spark.rdd.RDD
import org.apache.spark.shuffle.sort.SortShuffleManager
-import org.apache.spark.sql.execution.{CoalescedMapperPartitionSpec,
CoalescedPartitioner, CoalescedPartitionSpec, PartialMapperPartitionSpec,
PartialReducerPartitionSpec, ShufflePartitionSpec}
+import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.metric.{SQLMetric,
SQLShuffleReadMetricsReporter}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.vectorized.ColumnarBatch
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/IpcInputStreamIterator.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/IpcInputStreamIterator.scala
similarity index 95%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/IpcInputStreamIterator.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/IpcInputStreamIterator.scala
index d1d5af35..aa405504 100644
---
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/IpcInputStreamIterator.scala
+++
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/IpcInputStreamIterator.scala
@@ -19,12 +19,9 @@
package org.apache.spark.sql.comet.execution.shuffle
-import java.io.EOFException
-import java.io.InputStream
-import java.nio.ByteBuffer
-import java.nio.ByteOrder
-import java.nio.channels.Channels
-import java.nio.channels.ReadableByteChannel
+import java.io.{EOFException, InputStream}
+import java.nio.{ByteBuffer, ByteOrder}
+import java.nio.channels.{Channels, ReadableByteChannel}
import org.apache.spark.TaskContext
import org.apache.spark.internal.Logging
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ShuffleUtils.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ShuffleUtils.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ShuffleUtils.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ShuffleUtils.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
