LuciferYang commented on code in PR #37624:
URL: https://github.com/apache/spark/pull/37624#discussion_r956566370
##########
resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala:
##########
@@ -770,6 +771,40 @@ abstract class YarnShuffleServiceSuite extends
SparkFunSuite with Matchers {
s1.stop()
}
+ test("SPARK-40186: shuffleMergeManager should have been shutdown before db
closed") {
+ val maxId = 100
+ s1 = createYarnShuffleService()
+ val resolver = s1.shuffleMergeManager.asInstanceOf[RemoteBlockPushResolver]
+ (0 to maxId).foreach { id =>
+ val appId = ApplicationId.newInstance(0, id)
+ val appInfo = makeAppInfo("user", appId)
+ s1.initializeApplication(appInfo)
+ val mergedShuffleInfo =
+ new ExecutorShuffleInfo(
+ Array(new File(tempDir, "foo/foo").getAbsolutePath,
+ new File(tempDir, "bar/bar").getAbsolutePath), 3,
+ SORT_MANAGER_WITH_MERGE_SHUFFLE_META_WithAttemptID1)
+ resolver.registerExecutor(appId.toString, mergedShuffleInfo)
+ }
+
+ (0 until maxId).foreach { id =>
+ val appId = ApplicationId.newInstance(0, id)
+ resolver.applicationRemoved(appId.toString, true)
+ }
+
+ s1.stop()
+
+ assert(ShuffleTestAccessor.isMergedShuffleCleanerShutdown(resolver))
+ assert(ShuffleTestAccessor.mergeManagerLevelDB(resolver) == null)
+
+ val message = intercept[RejectedExecutionException] {
Review Comment:
> If we want to keep the final, I think of a way：
>
> Add a boolean type closed status to `shuffledb.LevelDB` and
`shuffledb.RocksDB` and set closed to true when `shuffledb.*DB.close()`, then
check closed is false before each api call and throw an `IllegalStateException`
if check fail
>
> Do you think this would be better? @mridulm
@mridulm I test this way, it seems work, but since methods
`removeAppAttemptPathInfoFromDB ` and `removeOldApplicationAttemptsFromDb `
catch all `Exception` now, so the above test scenario will sill catch
`RejectedExecutionException `, not `IllegalStateException`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
