[
https://issues.apache.org/jira/browse/AURORA-584?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14064433#comment-14064433
]
Jay Buffington commented on AURORA-584:
---------------------------------------
[~bhuvan] pointed out to me that this is likely when the scheduler has failed
over. It does this once every hour or so when the replicated log is being
flushed. The solution is to use the active master.
I propose two changes:
* when a slave is not active do a 302 redirect to the active one
* provide a status page I can point a load balancer at so the load balancer
can always send me to the active master
> Storage is not READY uncaught exception during getJobSummary
> ------------------------------------------------------------
>
> Key: AURORA-584
> URL: https://issues.apache.org/jira/browse/AURORA-584
> Project: Aurora
> Issue Type: Bug
> Reporter: Jay Buffington
>
> When loading http://aurora-scheduler:8081/scheduler/jaybuff/devel/memcached I
> get the following error in the response:
> {noformat}
> An error occurred when querying the server. Please reload this page.
> Error fetching tasks: No error message returned by the scheduler
> {noformat}
> I've seen this message enough to be annoying over the last month or two. I'm
> currently seeing it while running commit 84ee1571 (with a couple of simple
> and (hopefully) unrelated branches applied to it).
> Generally I wait a minute and reload the /scheduler page a couple of times
> and the issue goes away.
> tailing /var/log/aurora/aurora-scheduler.log on the scheduler I see the
> following stacktrace at the time of the error:
> {noformat}
> 0716 16:30:15.008 THREAD140
> org.apache.aurora.scheduler.thrift.aop.LoggingInterceptor.invoke: Uncaught
> exception while handling getJobSummary(jaybuff)
> org.apache.aurora.scheduler.storage.Storage$StorageException: Storage is not
> READY
> at
> org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.checkInState(CallOrderEnforcingStorage.java:78)
> at
> org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.weaklyConsistentRead(CallOrderEnforcingStorage.java:122)
> at
> org.apache.aurora.scheduler.storage.Storage$Util.weaklyConsistentFetchTasks(Storage.java:295)
> at
> org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.getTasks(SchedulerThriftInterface.java:584)
> at
> org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.getJobSummary(SchedulerThriftInterface.java:511)
> at
> org.apache.aurora.scheduler.thrift.aop.ServerInfoInterceptor.invoke(ServerInfoInterceptor.java:32)
> at
> org.apache.aurora.scheduler.thrift.aop.ThriftStatsExporterInterceptor.invoke(ThriftStatsExporterInterceptor.java:47)
> at
> org.apache.aurora.scheduler.thrift.aop.FeatureToggleInterceptor.invoke(FeatureToggleInterceptor.java:38)
> at
> org.apache.aurora.scheduler.thrift.aop.LoggingInterceptor.invoke(LoggingInterceptor.java:87)
> at
> org.apache.aurora.gen.ReadOnlyScheduler$Processor$getJobSummary.getResult(ReadOnlyScheduler.java:723)
> at
> org.apache.aurora.gen.ReadOnlyScheduler$Processor$getJobSummary.getResult(ReadOnlyScheduler.java:708)
> at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
> at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
> at org.apache.thrift.server.TServlet.doPost(TServlet.java:83)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> at
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:66)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at
> com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at
> com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118)
> at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at org.mortbay.servlet.UserAgentFilter.doFilter(UserAgentFilter.java:78)
> at org.mortbay.servlet.GzipFilter.doFilter(GzipFilter.java:131)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> at
> org.mortbay.jetty.handler.RequestLogHandler.handle(RequestLogHandler.java:49)
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> at org.mortbay.jetty.Server.handle(Server.java:326)
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:943)
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
> at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
> at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.2#6252)
