Copilot commented on code in PR #13599:
URL: https://github.com/apache/gluten/pull/13599#discussion_r4550223368


##########
backends-velox/src-kafka/test/scala/org/apache/spark/sql/execution/benchmark/KafkaIntegrationBenchmark.scala:
##########
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.gluten.execution.kafka.KafkaTestUtils
+
+import org.apache.spark.SparkConf
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.execution.datasources.v2.MicroBatchScanExec
+import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryListener, 
Trigger}
+
+import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig, 
NewTopic}
+
+import java.io.File
+import java.time.Duration
+import java.util.{Collections, Properties, UUID}
+import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
+
+import scala.collection.JavaConverters._
+
+/**
+ * Benchmark the supported Velox Kafka read integration, fallback, checkpoint 
recovery and a bounded
+ * Spark-producer-to-Parquet pipeline. Native reads use micro-batch execution; 
Kafka writes and
+ * checkpoint management remain Spark operations.
+ */
+object KafkaIntegrationBenchmark extends SqlBasedBenchmark {
+  private val scenarioNames = Seq("scan", "projection", "headers", 
"checkpoint", "pipeline")
+  private val kafkaColumns =
+    Seq("key", "value", "topic", "partition", "offset", "timestamp", 
"timestampType")
+
+  private case class Settings(
+      broker: String,
+      rows: Long,
+      partitions: Int,
+      payloadBytes: Int,
+      maxOffsets: Long,
+      iterations: Int,
+      timeoutMs: Long,
+      engines: Seq[Boolean])
+
+  private case class ReadResult(
+      queryId: UUID,
+      runId: UUID,
+      samples: Seq[KafkaBenchmarkSample],
+      elapsedMs: Double,
+      plan: String)
+
+  override def getSparkSession: SparkSession = {
+    val conf = new SparkConf()
+      .setAppName("KafkaIntegrationBenchmark")
+      .setIfMissing("spark.master", "local[2]")
+      .setIfMissing("spark.sql.shuffle.partitions", "2")
+      .setIfMissing("spark.sql.adaptive.enabled", "false")
+      .setIfMissing("spark.sql.streaming.stopTimeout", "10000")
+    if (conf.get("spark.gluten.benchmark.kafka.engine", "both") != "spark") {
+      conf
+        .setIfMissing("spark.plugins", "org.apache.gluten.GlutenPlugin")
+        .setIfMissing(
+          "spark.shuffle.manager",
+          "org.apache.spark.shuffle.sort.ColumnarShuffleManager")
+        .setIfMissing("spark.memory.offHeap.enabled", "true")
+        .setIfMissing("spark.memory.offHeap.size", "1g")
+    }
+    SparkSession.builder().config(conf).getOrCreate()
+  }

Review Comment:
   Same comparability issue as the rate benchmark: with `engine=both` (the 
default here), the SparkSession is created with the Gluten plugin installed, 
and the “Spark” measurements are run by toggling `spark.gluten.enabled=false`. 
If the goal is a clean Spark baseline, this should use a separate 
SparkSession/JVM without `spark.plugins` set, or explicitly document that 
baseline numbers are taken with the plugin present but disabled.



##########
backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/RateMicroBatchBenchmark.scala:
##########
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryListener, 
StreamingQueryProgress, Trigger}
+
+import java.nio.charset.StandardCharsets.UTF_8
+import java.time.Instant
+import java.util.UUID
+import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
+
+import scala.collection.JavaConverters._
+
+/**
+ * A bounded rate-source baseline. This benchmark does not read Kafka or prove 
native scan support.
+ */
+object RateMicroBatchBenchmark extends SqlBasedBenchmark {
+  private def prefix: String = "spark.gluten.benchmark.rate."
+
+  override def getSparkSession: SparkSession = {
+    val conf = new SparkConf()
+      .setAppName("RateMicroBatchBenchmark")
+      .setIfMissing("spark.master", "local[2]")
+      .setIfMissing("spark.sql.adaptive.enabled", "false")
+      .setIfMissing("spark.sql.streaming.stopTimeout", "10000")
+    if (conf.get(prefix + "engine", "spark") != "spark") {
+      conf
+        .setIfMissing("spark.plugins", "org.apache.gluten.GlutenPlugin")
+        .setIfMissing("spark.memory.offHeap.enabled", "true")
+        .setIfMissing("spark.memory.offHeap.size", "1g")
+    }
+    SparkSession.builder().config(conf).getOrCreate()
+  }

Review Comment:
   When `engine=both`, this constructs a single SparkSession with the Gluten 
plugin installed (because `engine != "spark"`), and then toggles 
`spark.gluten.enabled` per-iteration. That makes the “Spark” baseline 
potentially non-comparable because the plugin remains loaded (and may still 
affect planning/extensions even when disabled). If you want an apples-to-apples 
baseline, either (1) run Spark and Gluten in separate JVM/SparkSessions, or (2) 
clearly document that the Spark baseline here is “plugin-loaded but disabled” 
and interpret results accordingly.



##########
gluten-ut/spark35/src-kafka/test/scala/org/apache/spark/sql/kafka010/GlutenKafkaMicroBatchV2SourceSuite.scala:
##########
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.kafka010
+
+import org.apache.gluten.execution.MicroBatchScanExecTransformer
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.datasources.v2.MicroBatchScanExec
+import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
+import org.apache.spark.sql.execution.streaming.{SerializedOffset, 
StreamingQueryWrapper}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.Trigger
+
+import org.apache.kafka.common.TopicPartition
+
+import java.io.File
+import java.nio.charset.StandardCharsets.UTF_8
+
+class GlutenKafkaMicroBatchV2SourceSuite
+  extends KafkaMicroBatchV2SourceSuite
+  with GlutenKafkaSourceTest {
+
+  import testImplicits._
+
+  testGluten("Kafka stream-stream self-join preserves output and a single 
source offset") {
+    // Preserve Spark's self-join coverage without its unsupported 
StreamingQueryProgress row
+    // counts. Spark 3.5 ProgressReporter does not recognize native 
micro-batch scan nodes.
+    Seq(false, true).foreach {
+      reuseExchange =>
+        withSQLConf(SQLConf.EXCHANGE_REUSE_ENABLED.key -> 
reuseExchange.toString) {
+          val topic = newTopic()
+          testUtils.createTopic(topic, partitions = 2)
+          require(testUtils.getLatestOffsets(Set(topic)).size == 2)
+
+          val values = spark.readStream
+            .format("kafka")
+            .option("subscribe", topic)
+            .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+            .option("kafka.metadata.max.age.ms", "1")
+            .load()
+            .selectExpr(
+              "CAST(CAST(value AS STRING) AS INT) AS value",
+              "CAST(CAST(value AS STRING) AS INT) % 5 AS key")
+
+          testStream(values.join(values, "key"))(
+            makeSureGetOffsetCalled,
+            AddKafkaData(Set(topic), 1, 2),
+            CheckAnswer((1, 1, 1), (2, 2, 2)),
+            AddKafkaData(Set(topic), 6, 3),
+            CheckAnswer((1, 1, 1), (2, 2, 2), (3, 3, 3), (1, 6, 1), (1, 1, 6), 
(1, 6, 6)),
+            AssertOnQuery {
+              query =>
+                assert(query.availableOffsets.iterator.size == 1)
+                val expectedOffset = 
query.availableOffsets.iterator.next()._2.json
+                val loggedOffsets = query.offsetLog.getLatest
+                  .getOrElse(fail("No Kafka offset log entry found"))
+                  ._2.offsets
+                // Spark stores one checkpoint entry per logical source 
occurrence. Both sides
+                // of this self-join must record the same offset for the 
single shared source.
+                assert(loggedOffsets.size == 2)
+                assert(loggedOffsets.forall(_.exists(_.json == 
expectedOffset)))
+                assertKafkaScan(query, requireScan = true)
+                val plan = query.lastExecution.executedPlan
+                
assert(plan.exists(_.isInstanceOf[MicroBatchScanExecTransformer]))
+                val reusedExchanges = plan.collect { case exchange: 
ReusedExchangeExec => exchange }
+                assert(reusedExchanges.size == (if (reuseExchange) 1 else 0))
+                true
+            }
+          )
+        }
+    }
+  }
+
+  testGluten("Kafka micro-batches consume records through a native scan") {
+    val topic = newTopic()
+    testUtils.createTopic(topic, partitions = 2)
+    val values = (0 until 12).map(_.toString)
+    testUtils.sendMessages(topic, values.toArray)
+
+    val input = spark.readStream
+      .format("kafka")
+      .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+      .option("subscribe", topic)
+      .option("startingOffsets", "earliest")
+      .load()
+      .selectExpr("CAST(value AS STRING)")
+
+    testStream(input)(
+      // Records were sent before testStream, so CheckAnswer has no AddData 
offset to await.
+      makeSureGetOffsetCalled,
+      CheckAnswer(values: _*),
+      AssertOnQuery {
+        query =>
+          assertKafkaScan(query, requireScan = true)
+          assert(
+            
query.lastExecution.executedPlan.exists(_.isInstanceOf[MicroBatchScanExecTransformer]))
+          true
+      }
+    )
+  }
+
+  testGluten("Kafka retained headers use Spark fallback and preserve their 
values") {
+    val topic = newTopic()
+    testUtils.createTopic(topic, partitions = 1)
+    val headerValue = "header-value".getBytes(UTF_8)
+    testUtils.sendMessage(
+      new RecordBuilder(topic, "value")
+        .headers(Seq("header-key" -> headerValue))
+        .build())
+
+    val input = spark.readStream
+      .format("kafka")
+      .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+      .option("subscribe", topic)
+      .option("startingOffsets", "earliest")
+      .option("includeHeaders", "true")
+      .load()
+      .selectExpr("CAST(value AS STRING)", "headers")
+
+    testStream(input)(
+      // Wait for the preloaded record before checking its value and retained 
headers.
+      makeSureGetOffsetCalled,
+      CheckAnswer(Row("value", Seq(Row("header-key", headerValue)))),
+      AssertOnQuery {
+        query =>
+          assertKafkaScan(query, requireScan = true)
+          val plan = query.lastExecution.executedPlan
+          assert(!plan.exists(_.isInstanceOf[MicroBatchScanExecTransformer]))
+          assert(plan.exists {
+            case scan: MicroBatchScanExec =>
+              MicroBatchScanExecTransformer.supportsBatchScan(scan.scan)
+            case _ => false
+          })
+          true
+      }
+    )
+  }
+
+  testGluten("Kafka AvailableNow scans write Parquet and restart from the same 
checkpoint") {
+    val topic = newTopic()
+    val topicPartition = new TopicPartition(topic, 0)
+    testUtils.createTopic(topic, partitions = 1)
+
+    withTempDir {
+      dir =>
+        val outputPath = new File(dir, "parquet").getCanonicalPath
+        val checkpointPath = new File(dir, "checkpoint").getCanonicalPath
+        val input = spark.readStream
+          .format("kafka")
+          .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+          .option("subscribe", topic)
+          .option("startingOffsets", "earliest")
+          .option("maxOffsetsPerTrigger", 2)
+          .load()
+          .selectExpr("CAST(value AS STRING)")
+
+        def runAvailableNow(startOffset: Long, endOffset: Long): Unit = {
+          val query = input.writeStream
+            .format("parquet")
+            .outputMode("append")
+            .option("checkpointLocation", checkpointPath)
+            .trigger(Trigger.AvailableNow())
+            .start(outputPath)
+          try {
+            assert(query.awaitTermination(streamingTimeout.toMillis), 
"AvailableNow timed out")
+            val execution = 
query.asInstanceOf[StreamingQueryWrapper].streamingQuery
+            assertKafkaScan(execution, requireScan = true)
+            assert(
+              execution.lastExecution.executedPlan
+                .exists(_.isInstanceOf[MicroBatchScanExecTransformer]))
+
+            // Offset progress remains meaningful even when Spark cannot count 
native scan rows.
+            val batchEnds = query.recentProgress
+              .flatMap(_.sources)
+              .map {
+                source =>
+                  KafkaSourceOffset(SerializedOffset(source.endOffset))
+                    .partitionToOffsets(topicPartition)
+              }
+              .distinct
+              .toSeq
+            assert(batchEnds.nonEmpty, "AvailableNow reported no Kafka offset 
progress")
+            assert(batchEnds.last == endOffset)
+            val batchSizes = (Seq(startOffset) ++ batchEnds).sliding(2).map {
+              offsets => offsets.last - offsets.head
+            }.toSeq

Review Comment:
   `distinct.toSeq` does not guarantee ordering, so `batchEnds.last` (and the 
derived `batchSizes`) can be nondeterministic if the underlying collection 
order changes. To make this deterministic and ensure calculations reflect 
monotonic offset progression, sort `batchEnds` (e.g., increasing) before using 
`.last` and `.sliding(2)`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to