MonkeyCanCode commented on code in PR #59952:
URL: https://github.com/apache/spark/pull/59952#discussion_r4429659109
##########
sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala:
##########
@@ -133,6 +133,21 @@ private[sql] class SessionState(
*/
def clone(newSparkSession: SparkSession): SessionState =
createClone(newSparkSession, this)
+ /**
+ * Release the resources owned by this `SessionState`: the catalogs loaded
by its
+ * [[CatalogManager]], then the session artifacts. Catalogs are closed first
since they may have
+ * been loaded by the session classloader, which is closed with the
artifacts. Clones have their
+ * own catalogs and artifacts, so closing a `SessionState` does not affect
its clones. No catalog
+ * can be loaded afterwards.
+ */
+ def close(): Unit = {
+ try {
+ catalogManager.close()
Review Comment:
This is fixed.
##########
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala:
##########
@@ -439,6 +435,15 @@ case class SessionHolder(userId: String, sessionId:
String, session: SparkSessio
// Clean up ML cache (only if ML models were created)
mlCache.close()
+ // Close the catalogs loaded by this session, then clean up all artifacts.
Catalogs are closed
+ // first because they may have been loaded by the session classloader,
which is closed with
+ // the artifacts. Otherwise resources held by the catalogs (e.g. thread
pools) keep the
+ // session classloader alive. Cloned sessions own their catalogs and
artifacts, so this does
+ // not affect them.
+ // Note: interrupted executions may still be running; they fail if they
try to load a catalog
+ // after this point. There can also be concurrent AddArtifact calls still
adding something.
+ session.sessionState.close()
Review Comment:
This is fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
