mridulm commented on code in PR #37624:
URL: https://github.com/apache/spark/pull/37624#discussion_r956891830
##########
resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala:
##########
@@ -770,6 +771,40 @@ abstract class YarnShuffleServiceSuite extends
SparkFunSuite with Matchers {
s1.stop()
}
+ test("SPARK-40186: shuffleMergeManager should have been shutdown before db
closed") {
+ val maxId = 100
+ s1 = createYarnShuffleService()
+ val resolver = s1.shuffleMergeManager.asInstanceOf[RemoteBlockPushResolver]
+ (0 to maxId).foreach { id =>
+ val appId = ApplicationId.newInstance(0, id)
+ val appInfo = makeAppInfo("user", appId)
+ s1.initializeApplication(appInfo)
+ val mergedShuffleInfo =
+ new ExecutorShuffleInfo(
+ Array(new File(tempDir, "foo/foo").getAbsolutePath,
+ new File(tempDir, "bar/bar").getAbsolutePath), 3,
+ SORT_MANAGER_WITH_MERGE_SHUFFLE_META_WithAttemptID1)
+ resolver.registerExecutor(appId.toString, mergedShuffleInfo)
+ }
+
+ (0 until maxId).foreach { id =>
+ val appId = ApplicationId.newInstance(0, id)
+ resolver.applicationRemoved(appId.toString, true)
+ }
+
+ s1.stop()
+
+ assert(ShuffleTestAccessor.isMergedShuffleCleanerShutdown(resolver))
+ assert(ShuffleTestAccessor.mergeManagerLevelDB(resolver) == null)
+
+ val message = intercept[RejectedExecutionException] {
Review Comment:
To clarify, as part of the tests we do check for the expected behavior (that
the threadpool has been shutdown) ... What we removed is to check for a
specific runtime exception being raised through a codepath which should not get
invoked by NM (as the service is being shutdown).
That sounds fine to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
