Copilot commented on code in PR #13599: URL: https://github.com/apache/gluten/pull/13599#discussion_r4550223368
########## backends-velox/src-kafka/test/scala/org/apache/spark/sql/execution/benchmark/KafkaIntegrationBenchmark.scala: ########## @@ -0,0 +1,530 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.benchmark + +import org.apache.gluten.execution.kafka.KafkaTestUtils + +import org.apache.spark.SparkConf +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.execution.datasources.v2.MicroBatchScanExec +import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryListener, Trigger} + +import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig, NewTopic} + +import java.io.File +import java.time.Duration +import java.util.{Collections, Properties, UUID} +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit} + +import scala.collection.JavaConverters._ + +/** + * Benchmark the supported Velox Kafka read integration, fallback, checkpoint recovery and a bounded + * Spark-producer-to-Parquet pipeline. Native reads use micro-batch execution; Kafka writes and + * checkpoint management remain Spark operations. + */ +object KafkaIntegrationBenchmark extends SqlBasedBenchmark { + private val scenarioNames = Seq("scan", "projection", "headers", "checkpoint", "pipeline") + private val kafkaColumns = + Seq("key", "value", "topic", "partition", "offset", "timestamp", "timestampType") + + private case class Settings( + broker: String, + rows: Long, + partitions: Int, + payloadBytes: Int, + maxOffsets: Long, + iterations: Int, + timeoutMs: Long, + engines: Seq[Boolean]) + + private case class ReadResult( + queryId: UUID, + runId: UUID, + samples: Seq[KafkaBenchmarkSample], + elapsedMs: Double, + plan: String) + + override def getSparkSession: SparkSession = { + val conf = new SparkConf() + .setAppName("KafkaIntegrationBenchmark") + .setIfMissing("spark.master", "local[2]") + .setIfMissing("spark.sql.shuffle.partitions", "2") + .setIfMissing("spark.sql.adaptive.enabled", "false") + .setIfMissing("spark.sql.streaming.stopTimeout", "10000") + if (conf.get("spark.gluten.benchmark.kafka.engine", "both") != "spark") { + conf + .setIfMissing("spark.plugins", "org.apache.gluten.GlutenPlugin") + .setIfMissing( + "spark.shuffle.manager", + "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + .setIfMissing("spark.memory.offHeap.enabled", "true") + .setIfMissing("spark.memory.offHeap.size", "1g") + } + SparkSession.builder().config(conf).getOrCreate() + } Review Comment: Same comparability issue as the rate benchmark: with `engine=both` (the default here), the SparkSession is created with the Gluten plugin installed, and the “Spark” measurements are run by toggling `spark.gluten.enabled=false`. If the goal is a clean Spark baseline, this should use a separate SparkSession/JVM without `spark.plugins` set, or explicitly document that baseline numbers are taken with the plugin present but disabled. ########## backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/RateMicroBatchBenchmark.scala: ########## @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.benchmark + +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryListener, StreamingQueryProgress, Trigger} + +import java.nio.charset.StandardCharsets.UTF_8 +import java.time.Instant +import java.util.UUID +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit} + +import scala.collection.JavaConverters._ + +/** + * A bounded rate-source baseline. This benchmark does not read Kafka or prove native scan support. + */ +object RateMicroBatchBenchmark extends SqlBasedBenchmark { + private def prefix: String = "spark.gluten.benchmark.rate." + + override def getSparkSession: SparkSession = { + val conf = new SparkConf() + .setAppName("RateMicroBatchBenchmark") + .setIfMissing("spark.master", "local[2]") + .setIfMissing("spark.sql.adaptive.enabled", "false") + .setIfMissing("spark.sql.streaming.stopTimeout", "10000") + if (conf.get(prefix + "engine", "spark") != "spark") { + conf + .setIfMissing("spark.plugins", "org.apache.gluten.GlutenPlugin") + .setIfMissing("spark.memory.offHeap.enabled", "true") + .setIfMissing("spark.memory.offHeap.size", "1g") + } + SparkSession.builder().config(conf).getOrCreate() + } Review Comment: When `engine=both`, this constructs a single SparkSession with the Gluten plugin installed (because `engine != "spark"`), and then toggles `spark.gluten.enabled` per-iteration. That makes the “Spark” baseline potentially non-comparable because the plugin remains loaded (and may still affect planning/extensions even when disabled). If you want an apples-to-apples baseline, either (1) run Spark and Gluten in separate JVM/SparkSessions, or (2) clearly document that the Spark baseline here is “plugin-loaded but disabled” and interpret results accordingly. ########## gluten-ut/spark35/src-kafka/test/scala/org/apache/spark/sql/kafka010/GlutenKafkaMicroBatchV2SourceSuite.scala: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.kafka010 + +import org.apache.gluten.execution.MicroBatchScanExecTransformer + +import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.datasources.v2.MicroBatchScanExec +import org.apache.spark.sql.execution.exchange.ReusedExchangeExec +import org.apache.spark.sql.execution.streaming.{SerializedOffset, StreamingQueryWrapper} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.streaming.Trigger + +import org.apache.kafka.common.TopicPartition + +import java.io.File +import java.nio.charset.StandardCharsets.UTF_8 + +class GlutenKafkaMicroBatchV2SourceSuite + extends KafkaMicroBatchV2SourceSuite + with GlutenKafkaSourceTest { + + import testImplicits._ + + testGluten("Kafka stream-stream self-join preserves output and a single source offset") { + // Preserve Spark's self-join coverage without its unsupported StreamingQueryProgress row + // counts. Spark 3.5 ProgressReporter does not recognize native micro-batch scan nodes. + Seq(false, true).foreach { + reuseExchange => + withSQLConf(SQLConf.EXCHANGE_REUSE_ENABLED.key -> reuseExchange.toString) { + val topic = newTopic() + testUtils.createTopic(topic, partitions = 2) + require(testUtils.getLatestOffsets(Set(topic)).size == 2) + + val values = spark.readStream + .format("kafka") + .option("subscribe", topic) + .option("kafka.bootstrap.servers", testUtils.brokerAddress) + .option("kafka.metadata.max.age.ms", "1") + .load() + .selectExpr( + "CAST(CAST(value AS STRING) AS INT) AS value", + "CAST(CAST(value AS STRING) AS INT) % 5 AS key") + + testStream(values.join(values, "key"))( + makeSureGetOffsetCalled, + AddKafkaData(Set(topic), 1, 2), + CheckAnswer((1, 1, 1), (2, 2, 2)), + AddKafkaData(Set(topic), 6, 3), + CheckAnswer((1, 1, 1), (2, 2, 2), (3, 3, 3), (1, 6, 1), (1, 1, 6), (1, 6, 6)), + AssertOnQuery { + query => + assert(query.availableOffsets.iterator.size == 1) + val expectedOffset = query.availableOffsets.iterator.next()._2.json + val loggedOffsets = query.offsetLog.getLatest + .getOrElse(fail("No Kafka offset log entry found")) + ._2.offsets + // Spark stores one checkpoint entry per logical source occurrence. Both sides + // of this self-join must record the same offset for the single shared source. + assert(loggedOffsets.size == 2) + assert(loggedOffsets.forall(_.exists(_.json == expectedOffset))) + assertKafkaScan(query, requireScan = true) + val plan = query.lastExecution.executedPlan + assert(plan.exists(_.isInstanceOf[MicroBatchScanExecTransformer])) + val reusedExchanges = plan.collect { case exchange: ReusedExchangeExec => exchange } + assert(reusedExchanges.size == (if (reuseExchange) 1 else 0)) + true + } + ) + } + } + } + + testGluten("Kafka micro-batches consume records through a native scan") { + val topic = newTopic() + testUtils.createTopic(topic, partitions = 2) + val values = (0 until 12).map(_.toString) + testUtils.sendMessages(topic, values.toArray) + + val input = spark.readStream + .format("kafka") + .option("kafka.bootstrap.servers", testUtils.brokerAddress) + .option("subscribe", topic) + .option("startingOffsets", "earliest") + .load() + .selectExpr("CAST(value AS STRING)") + + testStream(input)( + // Records were sent before testStream, so CheckAnswer has no AddData offset to await. + makeSureGetOffsetCalled, + CheckAnswer(values: _*), + AssertOnQuery { + query => + assertKafkaScan(query, requireScan = true) + assert( + query.lastExecution.executedPlan.exists(_.isInstanceOf[MicroBatchScanExecTransformer])) + true + } + ) + } + + testGluten("Kafka retained headers use Spark fallback and preserve their values") { + val topic = newTopic() + testUtils.createTopic(topic, partitions = 1) + val headerValue = "header-value".getBytes(UTF_8) + testUtils.sendMessage( + new RecordBuilder(topic, "value") + .headers(Seq("header-key" -> headerValue)) + .build()) + + val input = spark.readStream + .format("kafka") + .option("kafka.bootstrap.servers", testUtils.brokerAddress) + .option("subscribe", topic) + .option("startingOffsets", "earliest") + .option("includeHeaders", "true") + .load() + .selectExpr("CAST(value AS STRING)", "headers") + + testStream(input)( + // Wait for the preloaded record before checking its value and retained headers. + makeSureGetOffsetCalled, + CheckAnswer(Row("value", Seq(Row("header-key", headerValue)))), + AssertOnQuery { + query => + assertKafkaScan(query, requireScan = true) + val plan = query.lastExecution.executedPlan + assert(!plan.exists(_.isInstanceOf[MicroBatchScanExecTransformer])) + assert(plan.exists { + case scan: MicroBatchScanExec => + MicroBatchScanExecTransformer.supportsBatchScan(scan.scan) + case _ => false + }) + true + } + ) + } + + testGluten("Kafka AvailableNow scans write Parquet and restart from the same checkpoint") { + val topic = newTopic() + val topicPartition = new TopicPartition(topic, 0) + testUtils.createTopic(topic, partitions = 1) + + withTempDir { + dir => + val outputPath = new File(dir, "parquet").getCanonicalPath + val checkpointPath = new File(dir, "checkpoint").getCanonicalPath + val input = spark.readStream + .format("kafka") + .option("kafka.bootstrap.servers", testUtils.brokerAddress) + .option("subscribe", topic) + .option("startingOffsets", "earliest") + .option("maxOffsetsPerTrigger", 2) + .load() + .selectExpr("CAST(value AS STRING)") + + def runAvailableNow(startOffset: Long, endOffset: Long): Unit = { + val query = input.writeStream + .format("parquet") + .outputMode("append") + .option("checkpointLocation", checkpointPath) + .trigger(Trigger.AvailableNow()) + .start(outputPath) + try { + assert(query.awaitTermination(streamingTimeout.toMillis), "AvailableNow timed out") + val execution = query.asInstanceOf[StreamingQueryWrapper].streamingQuery + assertKafkaScan(execution, requireScan = true) + assert( + execution.lastExecution.executedPlan + .exists(_.isInstanceOf[MicroBatchScanExecTransformer])) + + // Offset progress remains meaningful even when Spark cannot count native scan rows. + val batchEnds = query.recentProgress + .flatMap(_.sources) + .map { + source => + KafkaSourceOffset(SerializedOffset(source.endOffset)) + .partitionToOffsets(topicPartition) + } + .distinct + .toSeq + assert(batchEnds.nonEmpty, "AvailableNow reported no Kafka offset progress") + assert(batchEnds.last == endOffset) + val batchSizes = (Seq(startOffset) ++ batchEnds).sliding(2).map { + offsets => offsets.last - offsets.head + }.toSeq Review Comment: `distinct.toSeq` does not guarantee ordering, so `batchEnds.last` (and the derived `batchSizes`) can be nondeterministic if the underlying collection order changes. To make this deterministic and ensure calculations reflect monotonic offset progression, sort `batchEnds` (e.g., increasing) before using `.last` and `.sliding(2)`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
