[
https://issues.apache.org/jira/browse/SPARK-31247?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Hyukjin Kwon updated SPARK-31247:
---------------------------------
Description:
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120336/testReport/
{code}
Error Message
org.scalatest.exceptions.TestFailedException: Error adding data: Timeout after
waiting for 10000 ms.
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
scala.collection.TraversableLike.map(TraversableLike.scala:238)
scala.collection.TraversableLike.map$(TraversableLike.scala:231)
scala.collection.AbstractTraversable.map(Traversable.scala:108) == Progress
== AssertOnQuery(<condition>, ) AddKafkaData(topics = Set(topic-13), data
= WrappedArray(1, 2, 3), message = ) CheckAnswer: [2],[3],[4] StopStream
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
CheckAnswer: [2],[3],[4] StopStream AddKafkaData(topics =
Set(topic-13), data = WrappedArray(4, 5, 6), message = )
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
CheckAnswer: [2],[3],[4],[5],[6],[7] => AddKafkaData(topics =
Set(topic-13), data = WrappedArray(7, 8), message = ) CheckAnswer:
[2],[3],[4],[5],[6],[7],[8],[9] AssertOnQuery(<condition>, Add partitions)
AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13,
14, 15, 16), message = ) CheckAnswer:
[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17] ==
Stream == Output Mode: Append Stream state: {KafkaSource[Assign[topic-13-4,
topic-13-3, topic-13-2, topic-13-1, topic-13-0]]:
{"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}} Thread state: alive Thread stack
trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown
Source)
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown
Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown
Source) org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown
Source)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
== Sink == 0: 1: [3] [4] [2] 2: 3: 4: 5: [5] [7] [6] 6: == Plan == ==
Parsed Logical Plan == WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334] +- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
[cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
value#4323] +- StreamingDataSourceV2Relation [key#4308, value#4309,
topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Analyzed Logical
Plan == WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334] +- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
[cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
value#4323] +- StreamingDataSourceV2Relation [key#4308, value#4309,
topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Optimized
Logical Plan == WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334] +- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
[cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
value#4323] +- StreamingDataSourceV2Relation [key#4308, value#4309,
topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Physical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- *(1) SerializeFromObject [input[0, int, false] AS value#4334] +- *(1)
MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
obj#4333: int +- *(1) DeserializeToObject newInstance(class
scala.Tuple2), obj#4332: scala.Tuple2 +- *(1) Project [cast(key#4308
as string) AS key#4322, cast(value#4309 as string) AS value#4323]
+- *(1) Project [key#4308, value#4309, topic#4310, partition#4311,
offset#4312L, timestamp#4313, timestampType#4314] +-
ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L,
timestamp#4313, timestampType#4314] class
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
Stacktrace
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException:
Error adding data: Timeout after waiting for 10000 ms.
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
scala.collection.TraversableLike.map(TraversableLike.scala:238)
scala.collection.TraversableLike.map$(TraversableLike.scala:231)
scala.collection.AbstractTraversable.map(Traversable.scala:108)
== Progress ==
AssertOnQuery(<condition>, )
AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message =
)
CheckAnswer: [2],[3],[4]
StopStream
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
CheckAnswer: [2],[3],[4]
StopStream
AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message =
)
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
CheckAnswer: [2],[3],[4],[5],[6],[7]
=> AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )
CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
AssertOnQuery(<condition>, Add partitions)
AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13,
14, 15, 16), message = )
CheckAnswer:
[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]
== Stream ==
Output Mode: Append
Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2,
topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}}
Thread state: alive
Thread stack trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown
Source)
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown
Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown
Source)
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown
Source)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
== Sink ==
0:
1: [3] [4] [2]
2:
3:
4:
5: [5] [7] [6]
6:
== Plan ==
== Parsed Logical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
+- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
string) AS value#4323]
+- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
== Analyzed Logical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
+- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
string) AS value#4323]
+- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
== Optimized Logical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
+- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
string) AS value#4323]
+- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
== Physical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- *(1) SerializeFromObject [input[0, int, false] AS value#4334]
+- *(1) MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
obj#4333: int
+- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309
as string) AS value#4323]
+- *(1) Project [key#4308, value#4309, topic#4310, partition#4311,
offset#4312L, timestamp#4313, timestampType#4314]
+- ContinuousScan[key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
at
org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
at
org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
at
org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
at org.scalatest.Assertions.fail(Assertions.scala:1091)
at org.scalatest.Assertions.fail$(Assertions.scala:1087)
at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
at
org.apache.spark.sql.streaming.StreamTest.failTest$1(StreamTest.scala:448)
at
org.apache.spark.sql.streaming.StreamTest.executeAction$1(StreamTest.scala:720)
at
org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56(StreamTest.scala:774)
at
org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56$adapted(StreamTest.scala:761)
at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at
org.apache.spark.sql.streaming.StreamTest.liftedTree1$1(StreamTest.scala:761)
at
org.apache.spark.sql.streaming.StreamTest.testStream(StreamTest.scala:760)
at
org.apache.spark.sql.streaming.StreamTest.testStream$(StreamTest.scala:330)
at
org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:53)
at
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1728)
at
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.$anonfun$new$130(KafkaMicroBatchSourceSuite.scala:1312)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:151)
at
org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
at
org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:58)
at
org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
at
org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:58)
at
org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
at
org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
at org.scalatest.Suite.run(Suite.scala:1124)
at org.scalatest.Suite.run$(Suite.scala:1106)
at
org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
at
org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:58)
at
org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:58)
at
org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
at
org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
{code}
was:
Error Message
org.scalatest.exceptions.TestFailedException: Error adding data: Timeout after
waiting for 10000 ms.
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
scala.collection.TraversableLike.map(TraversableLike.scala:238)
scala.collection.TraversableLike.map$(TraversableLike.scala:231)
scala.collection.AbstractTraversable.map(Traversable.scala:108) == Progress
== AssertOnQuery(<condition>, ) AddKafkaData(topics = Set(topic-13), data
= WrappedArray(1, 2, 3), message = ) CheckAnswer: [2],[3],[4] StopStream
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
CheckAnswer: [2],[3],[4] StopStream AddKafkaData(topics =
Set(topic-13), data = WrappedArray(4, 5, 6), message = )
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
CheckAnswer: [2],[3],[4],[5],[6],[7] => AddKafkaData(topics =
Set(topic-13), data = WrappedArray(7, 8), message = ) CheckAnswer:
[2],[3],[4],[5],[6],[7],[8],[9] AssertOnQuery(<condition>, Add partitions)
AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13,
14, 15, 16), message = ) CheckAnswer:
[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17] ==
Stream == Output Mode: Append Stream state: {KafkaSource[Assign[topic-13-4,
topic-13-3, topic-13-2, topic-13-1, topic-13-0]]:
{"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}} Thread state: alive Thread stack
trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown
Source)
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown
Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown
Source) org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown
Source)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
== Sink == 0: 1: [3] [4] [2] 2: 3: 4: 5: [5] [7] [6] 6: == Plan == ==
Parsed Logical Plan == WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334] +- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
[cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
value#4323] +- StreamingDataSourceV2Relation [key#4308, value#4309,
topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Analyzed Logical
Plan == WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334] +- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
[cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
value#4323] +- StreamingDataSourceV2Relation [key#4308, value#4309,
topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Optimized
Logical Plan == WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334] +- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
[cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
value#4323] +- StreamingDataSourceV2Relation [key#4308, value#4309,
topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Physical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- *(1) SerializeFromObject [input[0, int, false] AS value#4334] +- *(1)
MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
obj#4333: int +- *(1) DeserializeToObject newInstance(class
scala.Tuple2), obj#4332: scala.Tuple2 +- *(1) Project [cast(key#4308
as string) AS key#4322, cast(value#4309 as string) AS value#4323]
+- *(1) Project [key#4308, value#4309, topic#4310, partition#4311,
offset#4312L, timestamp#4313, timestampType#4314] +-
ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L,
timestamp#4313, timestampType#4314] class
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
Stacktrace
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException:
Error adding data: Timeout after waiting for 10000 ms.
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
scala.collection.TraversableLike.map(TraversableLike.scala:238)
scala.collection.TraversableLike.map$(TraversableLike.scala:231)
scala.collection.AbstractTraversable.map(Traversable.scala:108)
== Progress ==
AssertOnQuery(<condition>, )
AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message =
)
CheckAnswer: [2],[3],[4]
StopStream
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
CheckAnswer: [2],[3],[4]
StopStream
AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message =
)
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
CheckAnswer: [2],[3],[4],[5],[6],[7]
=> AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )
CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
AssertOnQuery(<condition>, Add partitions)
AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13,
14, 15, 16), message = )
CheckAnswer:
[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]
== Stream ==
Output Mode: Append
Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2,
topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}}
Thread state: alive
Thread stack trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown
Source)
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown
Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown
Source)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown
Source)
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown
Source)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
== Sink ==
0:
1: [3] [4] [2]
2:
3:
4:
5: [5] [7] [6]
6:
== Plan ==
== Parsed Logical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
+- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
string) AS value#4323]
+- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
== Analyzed Logical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
+- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
string) AS value#4323]
+- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
== Optimized Logical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
+- MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
class scala.Tuple2, [StructField(_1,StringType,true),
StructField(_2,StringType,true)], obj#4333: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
string) AS value#4323]
+- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
== Physical Plan ==
WriteToContinuousDataSource
org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- *(1) SerializeFromObject [input[0, int, false] AS value#4334]
+- *(1) MapElements
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
obj#4333: int
+- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
scala.Tuple2
+- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309
as string) AS value#4323]
+- *(1) Project [key#4308, value#4309, topic#4310, partition#4311,
offset#4312L, timestamp#4313, timestampType#4314]
+- ContinuousScan[key#4308, value#4309, topic#4310,
partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
at
org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
at
org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
at
org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
at org.scalatest.Assertions.fail(Assertions.scala:1091)
at org.scalatest.Assertions.fail$(Assertions.scala:1087)
at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
at
org.apache.spark.sql.streaming.StreamTest.failTest$1(StreamTest.scala:448)
at
org.apache.spark.sql.streaming.StreamTest.executeAction$1(StreamTest.scala:720)
at
org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56(StreamTest.scala:774)
at
org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56$adapted(StreamTest.scala:761)
at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at
org.apache.spark.sql.streaming.StreamTest.liftedTree1$1(StreamTest.scala:761)
at
org.apache.spark.sql.streaming.StreamTest.testStream(StreamTest.scala:760)
at
org.apache.spark.sql.streaming.StreamTest.testStream$(StreamTest.scala:330)
at
org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:53)
at
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1728)
at
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.$anonfun$new$130(KafkaMicroBatchSourceSuite.scala:1312)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:151)
at
org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
at
org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:58)
at
org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
at
org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:58)
at
org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
at
org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
at org.scalatest.Suite.run(Suite.scala:1124)
at org.scalatest.Suite.run$(Suite.scala:1106)
at
org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
at
org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:58)
at
org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:58)
at
org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
at
org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
> Flaky test: KafkaContinuousSourceSuite.assign from latest offsets
> (failOnDataLoss: false)
> -----------------------------------------------------------------------------------------
>
> Key: SPARK-31247
> URL: https://issues.apache.org/jira/browse/SPARK-31247
> Project: Spark
> Issue Type: Bug
> Components: Structured Streaming, Tests
> Affects Versions: 3.0.0, 3.1.0
> Reporter: Gabor Somogyi
> Priority: Major
>
> https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120336/testReport/
> {code}
> Error Message
> org.scalatest.exceptions.TestFailedException: Error adding data: Timeout
> after waiting for 10000 ms.
> org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
>
> org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
>
> org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
> scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
> scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
> scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
> scala.collection.TraversableLike.map(TraversableLike.scala:238)
> scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> scala.collection.AbstractTraversable.map(Traversable.scala:108) == Progress
> == AssertOnQuery(<condition>, ) AddKafkaData(topics = Set(topic-13),
> data = WrappedArray(1, 2, 3), message = ) CheckAnswer: [2],[3],[4]
> StopStream
> StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
> CheckAnswer: [2],[3],[4] StopStream AddKafkaData(topics =
> Set(topic-13), data = WrappedArray(4, 5, 6), message = )
> StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
> CheckAnswer: [2],[3],[4],[5],[6],[7] => AddKafkaData(topics =
> Set(topic-13), data = WrappedArray(7, 8), message = ) CheckAnswer:
> [2],[3],[4],[5],[6],[7],[8],[9] AssertOnQuery(<condition>, Add partitions)
> AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12,
> 13, 14, 15, 16), message = ) CheckAnswer:
> [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17] ==
> Stream == Output Mode: Append Stream state: {KafkaSource[Assign[topic-13-4,
> topic-13-3, topic-13-2, topic-13-1, topic-13-0]]:
> {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}} Thread state: alive Thread
> stack trace: sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
> org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
> org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
> org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
> org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
>
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
> org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
> org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
>
> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
> org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown
> Source)
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
>
> org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown
> Source)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
>
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown
> Source)
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
>
> org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown
> Source)
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
>
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
>
> org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown
> Source) org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
>
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
>
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown
> Source)
> org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
>
> org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
>
> org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
>
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
>
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
>
> org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
>
> org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
> == Sink == 0: 1: [3] [4] [2] 2: 3: 4: 5: [5] [7] [6] 6: == Plan ==
> == Parsed Logical Plan == WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334] +-
> MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> class scala.Tuple2, [StructField(_1,StringType,true),
> StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
> newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
> [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
> value#4323] +- StreamingDataSourceV2Relation [key#4308,
> value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313,
> timestampType#4314],
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
> KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
> topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Analyzed
> Logical Plan == WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334] +-
> MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> class scala.Tuple2, [StructField(_1,StringType,true),
> StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
> newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
> [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
> value#4323] +- StreamingDataSourceV2Relation [key#4308,
> value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313,
> timestampType#4314],
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
> KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
> topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Optimized
> Logical Plan == WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334] +-
> MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> class scala.Tuple2, [StructField(_1,StringType,true),
> StructField(_2,StringType,true)], obj#4333: int +- DeserializeToObject
> newInstance(class scala.Tuple2), obj#4332: scala.Tuple2 +- Project
> [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS
> value#4323] +- StreamingDataSourceV2Relation [key#4308,
> value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313,
> timestampType#4314],
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
> KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
> topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}} == Physical Plan
> == WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- *(1) SerializeFromObject [input[0, int, false] AS value#4334] +- *(1)
> MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> obj#4333: int +- *(1) DeserializeToObject newInstance(class
> scala.Tuple2), obj#4332: scala.Tuple2 +- *(1) Project [cast(key#4308
> as string) AS key#4322, cast(value#4309 as string) AS value#4323]
> +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311,
> offset#4312L, timestamp#4313, timestampType#4314] +-
> ContinuousScan[key#4308, value#4309, topic#4310, partition#4311,
> offset#4312L, timestamp#4313, timestampType#4314] class
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
>
> Stacktrace
> sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException:
> Error adding data: Timeout after waiting for 10000 ms.
> org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
>
> org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
>
> org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
>
> scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
>
> scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
>
> scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
> scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
> scala.collection.TraversableLike.map(TraversableLike.scala:238)
> scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> scala.collection.AbstractTraversable.map(Traversable.scala:108)
> == Progress ==
> AssertOnQuery(<condition>, )
> AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message
> = )
> CheckAnswer: [2],[3],[4]
> StopStream
>
> StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
> CheckAnswer: [2],[3],[4]
> StopStream
> AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message
> = )
>
> StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
> CheckAnswer: [2],[3],[4],[5],[6],[7]
> => AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )
> CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
> AssertOnQuery(<condition>, Add partitions)
> AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12,
> 13, 14, 15, 16), message = )
> CheckAnswer:
> [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]
> == Stream ==
> Output Mode: Append
> Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2,
> topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}}
> Thread state: alive
> Thread stack trace: sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
> org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
> org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
> org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
> org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
> org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
> org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
> org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown
> Source)
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
> org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown
> Source)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown
> Source)
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
> org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown
> Source)
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
> org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown
> Source)
> org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown
> Source)
> org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
> org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
> org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
> org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
> org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
> == Sink ==
> 0:
> 1: [3] [4] [2]
> 2:
> 3:
> 4:
> 5: [5] [7] [6]
> 6:
> == Plan ==
> == Parsed Logical Plan ==
> WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334]
> +- MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> class scala.Tuple2, [StructField(_1,StringType,true),
> StructField(_2,StringType,true)], obj#4333: int
> +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
> scala.Tuple2
> +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
> string) AS value#4323]
> +- StreamingDataSourceV2Relation [key#4308, value#4309,
> topic#4310, partition#4311, offset#4312L, timestamp#4313,
> timestampType#4314],
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
> KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
> topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
> == Analyzed Logical Plan ==
> WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334]
> +- MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> class scala.Tuple2, [StructField(_1,StringType,true),
> StructField(_2,StringType,true)], obj#4333: int
> +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
> scala.Tuple2
> +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
> string) AS value#4323]
> +- StreamingDataSourceV2Relation [key#4308, value#4309,
> topic#4310, partition#4311, offset#4312L, timestamp#4313,
> timestampType#4314],
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
> KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
> topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
> == Optimized Logical Plan ==
> WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334]
> +- MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> class scala.Tuple2, [StructField(_1,StringType,true),
> StructField(_2,StringType,true)], obj#4333: int
> +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
> scala.Tuple2
> +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as
> string) AS value#4323]
> +- StreamingDataSourceV2Relation [key#4308, value#4309,
> topic#4310, partition#4311, offset#4312L, timestamp#4313,
> timestampType#4314],
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d,
> KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1,
> topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
> == Physical Plan ==
> WriteToContinuousDataSource
> org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- *(1) SerializeFromObject [input[0, int, false] AS value#4334]
> +- *(1) MapElements
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67,
> obj#4333: int
> +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332:
> scala.Tuple2
> +- *(1) Project [cast(key#4308 as string) AS key#4322,
> cast(value#4309 as string) AS value#4323]
> +- *(1) Project [key#4308, value#4309, topic#4310,
> partition#4311, offset#4312L, timestamp#4313, timestampType#4314]
> +- ContinuousScan[key#4308, value#4309, topic#4310,
> partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class
> org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
>
>
> at
> org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
> at
> org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
> at
> org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
> at org.scalatest.Assertions.fail(Assertions.scala:1091)
> at org.scalatest.Assertions.fail$(Assertions.scala:1087)
> at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
> at
> org.apache.spark.sql.streaming.StreamTest.failTest$1(StreamTest.scala:448)
> at
> org.apache.spark.sql.streaming.StreamTest.executeAction$1(StreamTest.scala:720)
> at
> org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56(StreamTest.scala:774)
> at
> org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56$adapted(StreamTest.scala:761)
> at
> scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at
> scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
> at
> org.apache.spark.sql.streaming.StreamTest.liftedTree1$1(StreamTest.scala:761)
> at
> org.apache.spark.sql.streaming.StreamTest.testStream(StreamTest.scala:760)
> at
> org.apache.spark.sql.streaming.StreamTest.testStream$(StreamTest.scala:330)
> at
> org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:53)
> at
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1728)
> at
> org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.$anonfun$new$130(KafkaMicroBatchSourceSuite.scala:1312)
> at
> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
> at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
> at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
> at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
> at org.scalatest.Transformer.apply(Transformer.scala:22)
> at org.scalatest.Transformer.apply(Transformer.scala:20)
> at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
> at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:151)
> at
> org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
> at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
> at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
> at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
> at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
> at
> org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:58)
> at
> org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
> at
> org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
> at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:58)
> at
> org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
> at
> org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
> at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
> at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
> at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
> at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
> at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
> at org.scalatest.Suite.run(Suite.scala:1124)
> at org.scalatest.Suite.run$(Suite.scala:1106)
> at
> org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
> at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
> at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
> at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
> at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
> at
> org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:58)
> at
> org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
> at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
> at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
> at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:58)
> at
> org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
> at
> org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
> at sbt.ForkMain$Run$2.call(ForkMain.java:296)
> at sbt.ForkMain$Run$2.call(ForkMain.java:286)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
