LuciferYang commented on code in PR #37624:
URL: https://github.com/apache/spark/pull/37624#discussion_r956849324
##########
resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala:
##########
@@ -770,6 +771,40 @@ abstract class YarnShuffleServiceSuite extends
SparkFunSuite with Matchers {
s1.stop()
}
+ test("SPARK-40186: shuffleMergeManager should have been shutdown before db
closed") {
+ val maxId = 100
+ s1 = createYarnShuffleService()
+ val resolver = s1.shuffleMergeManager.asInstanceOf[RemoteBlockPushResolver]
+ (0 to maxId).foreach { id =>
+ val appId = ApplicationId.newInstance(0, id)
+ val appInfo = makeAppInfo("user", appId)
+ s1.initializeApplication(appInfo)
+ val mergedShuffleInfo =
+ new ExecutorShuffleInfo(
+ Array(new File(tempDir, "foo/foo").getAbsolutePath,
+ new File(tempDir, "bar/bar").getAbsolutePath), 3,
+ SORT_MANAGER_WITH_MERGE_SHUFFLE_META_WithAttemptID1)
+ resolver.registerExecutor(appId.toString, mergedShuffleInfo)
+ }
+
+ (0 until maxId).foreach { id =>
+ val appId = ApplicationId.newInstance(0, id)
+ resolver.applicationRemoved(appId.toString, true)
+ }
+
+ s1.stop()
+
+ assert(ShuffleTestAccessor.isMergedShuffleCleanerShutdown(resolver))
+ assert(ShuffleTestAccessor.mergeManagerLevelDB(resolver) == null)
+
+ val message = intercept[RejectedExecutionException] {
Review Comment:
@mridulm
[0f2afe9](https://github.com/apache/spark/pull/37624/commits/0f2afe966336c63703cde1df6366db84d7a68ee9)
keep `DB` as final and remove the following asserts:
```
assert(ShuffleTestAccessor.mergeManagerLevelDB(resolver) == null)
val message = intercept[RejectedExecutionException] {
val appId = ApplicationId.newInstance(0, maxId)
resolver.applicationRemoved(appId.toString, true)
}.getMessage
assert(message.contains("Terminated, pool size = 0, " +
s"active threads = 0, queued tasks = 0, completed tasks = $maxId"))
```
Already file a new jira
[SPARK-40250](https://issues.apache.org/jira/browse/SPARK-40250) to further
check the issue mentioned above.
Do you think this is OK?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
