zhli1142015 commented on pull request #28769:
URL: https://github.com/apache/spark/pull/28769#issuecomment-641668316
> I'm not sure, I don't see any usages of it that don't close the iterator
in a finally block. I do wonder where these are allocated, if there is a
straightforward fix to the resource leak elsewhere. I don't see a scenario
where the DB can close before the iterator does, except maybe shutdown, not
sure.
>
> If it's reproducible, one interesting hack you could put in here is to
record the stack trace where the iterator is constructed temporarily, then see
what's left to close in your new code path, to debug where the leaked ones came
from?
here are some examples, all of them are from "AppStatusStore".
Iterator is not closed before db is closed: 1752491640
Iterator is not closed before db is closed: 686818478
Iterator is not closed before db is closed: 1061522397
Construct iterator: 1061522397. Stacktrace: java.lang.Throwable
at org.apache.spark.util.kvstore.LevelDB.getStackTrace(LevelDB.java:272)
at org.apache.spark.util.kvstore.LevelDB$1.iterator(LevelDB.java:206)
at
scala.collection.convert.Wrappers$JIterableWrapper.iterator(Wrappers.scala:55)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
**at
org.apache.spark.status.AppStatusStore.stageList(AppStatusStore.scala:102)**
at org.apache.spark.ui.jobs.AllStagesPage.render(AllStagesPage.scala:46)
at org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:89)
at org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:80)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at
org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
at
org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at
org.apache.spark.deploy.history.ApplicationCacheCheckFilter.doFilter(ApplicationCache.scala:404)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at
org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at
org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at
org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at
org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:753)
at
org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
at
org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.sparkproject.jetty.server.Server.handle(Server.java:505)
at
org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at
org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:103)
at
org.sparkproject.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
at
org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
at java.lang.Thread.run(Thread.java:748)
Construct iterator: 686818478. Stacktrace: java.lang.Throwable
at org.apache.spark.util.kvstore.LevelDB.getStackTrace(LevelDB.java:272)
at org.apache.spark.util.kvstore.LevelDB$1.iterator(LevelDB.java:206)
**at
org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:42)**
at org.apache.spark.ui.SparkUI.getSparkUser(SparkUI.scala:85)
at org.apache.spark.ui.SparkUI.getApplicationInfoList(SparkUI.scala:127)
at
org.apache.spark.deploy.history.ApplicationCache.org$apache$spark$deploy$history$ApplicationCache$$loadApplicationEntry(ApplicationCache.scala:176)
at
org.apache.spark.deploy.history.ApplicationCache$$anon$1.load(ApplicationCache.scala:56)
at
org.apache.spark.deploy.history.ApplicationCache$$anon$1.load(ApplicationCache.scala:52)
at
org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at
org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at
org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at
org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000)
at
org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at
org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at
org.apache.spark.deploy.history.ApplicationCache.get(ApplicationCache.scala:89)
at
org.apache.spark.deploy.history.ApplicationCache.withSparkUI(ApplicationCache.scala:101)
at
org.apache.spark.deploy.history.HistoryServer.org$apache$spark$deploy$history$HistoryServer$$loadAppUi(HistoryServer.scala:248)
at
org.apache.spark.deploy.history.HistoryServer$$anon$1.doGet(HistoryServer.scala:101)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at
org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
at
org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at
org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at
org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at
org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at
org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:753)
at
org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
at
org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.sparkproject.jetty.server.Server.handle(Server.java:505)
at
org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at
org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:103)
at
org.sparkproject.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
at
org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
at java.lang.Thread.run(Thread.java:748)
Construct iterator: 1752491640. Stacktrace: java.lang.Throwable
at org.apache.spark.util.kvstore.LevelDB.getStackTrace(LevelDB.java:272)
at org.apache.spark.util.kvstore.LevelDB$1.iterator(LevelDB.java:206)
**at
org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:42)**
at
org.apache.spark.deploy.history.HistoryAppStatusStore.<init>(HistoryAppStatusStore.scala:34)
at
org.apache.spark.deploy.history.FsHistoryProvider.getAppUI(FsHistoryProvider.scala:373)
at
org.apache.spark.deploy.history.HistoryServer.getAppUI(HistoryServer.scala:191)
at
org.apache.spark.deploy.history.ApplicationCache.$anonfun$loadApplicationEntry$2(ApplicationCache.scala:163)
at
org.apache.spark.deploy.history.ApplicationCache.time(ApplicationCache.scala:135)
at
org.apache.spark.deploy.history.ApplicationCache.org$apache$spark$deploy$history$ApplicationCache$$loadApplicationEntry(ApplicationCache.scala:161)
at
org.apache.spark.deploy.history.ApplicationCache$$anon$1.load(ApplicationCache.scala:56)
at
org.apache.spark.deploy.history.ApplicationCache$$anon$1.load(ApplicationCache.scala:52)
at
org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at
org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at
org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at
org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000)
at
org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at
org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at
org.apache.spark.deploy.history.ApplicationCache.get(ApplicationCache.scala:89)
at
org.apache.spark.deploy.history.ApplicationCache.withSparkUI(ApplicationCache.scala:101)
at
org.apache.spark.deploy.history.HistoryServer.org$apache$spark$deploy$history$HistoryServer$$loadAppUi(HistoryServer.scala:248)
at
org.apache.spark.deploy.history.HistoryServer$$anon$1.doGet(HistoryServer.scala:101)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at
org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
at
org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)
at
org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at
org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at
org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at
org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at
org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:753)
at
org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
at
org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.sparkproject.jetty.server.Server.handle(Server.java:505)
at
org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at
org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:103)
at
org.sparkproject.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
at
org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
at java.lang.Thread.run(Thread.java:748)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
