juliuszsompolski commented on code in PR #43745:
URL: https://github.com/apache/spark/pull/43745#discussion_r1389786137
##########
connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala:
##########
@@ -387,6 +387,33 @@ class SparkConnectClientSuite extends ConnectFunSuite with
BeforeAndAfterEach {
}
assert(dummyFn.counter == 2)
}
+
+ test("SPARK-45871: Client execute iterator.toSeq consumes the reattachable
iterator") {
+ startDummyServer(0)
+ client = SparkConnectClient
+ .builder()
+ .connectionString(s"sc://localhost:${server.getPort}")
+ .enableReattachableExecute()
+ .build()
+ val session = SparkSession.builder().client(client).create()
+ val cmd = session.newCommand(b =>
+ b.setSqlCommand(
+ proto.SqlCommand
+ .newBuilder()
+ .setSql("select * from range(10000000)")))
+ val plan = proto.Plan.newBuilder().setCommand(cmd)
+ val iter = client.execute(plan.build())
+ val reattachableIter =
+ ExecutePlanResponseReattachableIterator.fromIterator(iter)
+ iter.toSeq
+ // In several places in SparkSession, we depend on `.toSeq` to consume and
close the iterator.
+ // If this assertion fails, we need to double check the correctness of
that.
+ // In scala 2.12 `s.c.TraversableOnce#toSeq` builds an `immutable.Stream`,
+ // which is a tail lazy structure and this would fail.
+ // In scala 2.13 `s.c.IterableOnceOps#toSeq` builds an `immutable.Seq`
which is not
+ // lazy and will consume and close the iterator.
Review Comment:
to update to no longer using `.toSeq`:
```suggestion
iter.foreach(_ => ())
// Make sure that consuming the iterator closes the reattachableIter
inside.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
