Copilot commented on code in PR #1958:
URL: https://github.com/apache/auron/pull/1958#discussion_r2726254530
##########
spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffleExchangeExec.scala:
##########
@@ -126,12 +121,46 @@ case class NativeShuffleExchangeExec(
new
SQLShuffleWriteMetricsReporter(context.taskMetrics().shuffleWriteMetrics,
metrics)
}
+ @sparkver("3.1 / 3.2 / 3.3 / 3.4 / 3.5")
override def write(
rdd: RDD[_],
dep: ShuffleDependency[_, _, _],
mapId: Long,
context: TaskContext,
- partition: Partition): MapStatus = {
+ partition: Partition): org.apache.spark.scheduler.MapStatus = {
+ internalWrite(rdd, dep, mapId, context, partition)
+ }
+
+ @sparkver("4.1")
+ override def write(
+ inputs: Iterator[_],
+ dep: ShuffleDependency[_, _, _],
+ mapId: Long,
+ mapIndex: Int,
+ context: TaskContext): org.apache.spark.scheduler.MapStatus = {
+ import
org.apache.spark.sql.execution.auron.shuffle.AuronShuffleDependency
+
+ // SPARK-44605: Spark 4+ refines ShuffleWriteProcessor API, leading to
early execution of NativeRDD.ShuffleWrite iterator
+ // Adaptation: Return empty iterator in NativeRDD.compute() to defer
execution to ShuffleWriteProcessor.write() (align with Spark 3.x logic)
+ assert(
Review Comment:
Using `assert(inputs.isEmpty, ...)` for a runtime invariant is fragile
because assertions can be disabled/elided, in which case non-empty input would
be silently ignored. Prefer `require(inputs.isEmpty, ...)` or explicit error
handling so this fails deterministically if Spark changes the contract.
```suggestion
require(
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
