Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/20386#discussion_r164680632
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memoryV2.scala
---
@@ -118,14 +118,21 @@ class MemoryWriter(sink: MemorySinkV2, batchId: Long,
outputMode: OutputMode)
override def createWriterFactory: MemoryWriterFactory =
MemoryWriterFactory(outputMode)
- def commit(messages: Array[WriterCommitMessage]): Unit = {
+ private val messages = new ArrayBuffer[WriterCommitMessage]()
+
+ override def add(message: WriterCommitMessage): Unit = synchronized {
+ messages += message
+ }
+
+ def commit(): Unit = synchronized {
val newRows = messages.flatMap {
case message: MemoryWriterCommitMessage => message.data
- }
+ }.toArray
sink.write(batchId, outputMode, newRows)
+ messages.clear()
}
- override def abort(messages: Array[WriterCommitMessage]): Unit = {
+ override def abort(): Unit = {
--- End diff --
ditto
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
