[ https://issues.apache.org/jira/browse/KYLIN-4305?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Xiaoxiang Yu updated KYLIN-4305: -------------------------------- Attachment: after_repair_receiver.jstack > Streaming Receiver cannot limit income query request or cancel long-running > query > --------------------------------------------------------------------------------- > > Key: KYLIN-4305 > URL: https://issues.apache.org/jira/browse/KYLIN-4305 > Project: Kylin > Issue Type: Improvement > Components: Real-time Streaming > Reporter: Xiaoxiang Yu > Priority: Major > Attachments: Jietu20191217-221025.png, after_repair_receiver.jstack, > image-2019-12-17-22-12-01-098.png, streaming_receiver_jstack.log > > > When under heavy load(high rate of query request), receiver can not stand it, > and most quey may timeout, but the query processing thread cannot be > cancelled in receiver side, which will cause receiver's crash. You have to > restart it. > kylin.log > {code:java} > Caused by: java.lang.RuntimeException: timeout when call stream rpc > at > org.apache.kylin.storage.stream.rpc.HttpStreamDataSearchClient$QueuedStreamingTupleIterator.hasNext(HttpStreamDataSearchClient.java:298) > at com.google.common.collect.Iterators$5.hasNext(Iterators.java:596) > at > org.apache.kylin.metadata.tuple.CompoundTupleIterator.hasNext(CompoundTupleIterator.java:52) > at > org.apache.kylin.query.enumerator.OLAPEnumerator.moveNext(OLAPEnumerator.java:63) > at Baz$1$1.moveNext(Unknown Source) > at > org.apache.calcite.linq4j.EnumerableDefaults.groupBy_(EnumerableDefaults.java:825) > at > org.apache.calcite.linq4j.EnumerableDefaults.groupBy(EnumerableDefaults.java:761) > at > org.apache.calcite.linq4j.DefaultEnumerable.groupBy(DefaultEnumerable.java:302) > at Baz.bind(Unknown Source) > at > org.apache.calcite.jdbc.CalcitePrepare$CalciteSignature.enumerable(CalcitePrepare.java:365) > at > org.apache.calcite.jdbc.CalciteConnectionImpl.enumerable(CalciteConnectionImpl.java:301) > at > org.apache.calcite.jdbc.CalciteMetaImpl._createIterable(CalciteMetaImpl.java:559) > at > org.apache.calcite.jdbc.CalciteMetaImpl.createIterable(CalciteMetaImpl.java:550) > at > org.apache.calcite.avatica.AvaticaResultSet.execute(AvaticaResultSet.java:182) > at > org.apache.calcite.jdbc.CalciteResultSet.execute(CalciteResultSet.java:67) > at > org.apache.calcite.jdbc.CalciteResultSet.execute(CalciteResultSet.java:44) > at > org.apache.calcite.avatica.AvaticaConnection$1.execute(AvaticaConnection.java:667) > at > org.apache.calcite.jdbc.CalciteMetaImpl.prepareAndExecute(CalciteMetaImpl.java:619) > at > org.apache.calcite.avatica.AvaticaConnection.prepareAndExecuteInternal(AvaticaConnection.java:675) > at > org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:156) > ... 83 more > {code} > jstack > {code:java} > "qtp1901663135-48" #48 prio=5 os_prio=0 tid=0x00007f839995f800 nid=0x3cdd > runnable [0x00007f83674fe000] > java.lang.Thread.State: RUNNABLE > at java.lang.Thread.yield(Native Method) > at > org.apache.kylin.stream.core.query.MultiThreadsResultCollector$1.hasNext(MultiThreadsResultCollector.java:75) > at > org.apache.kylin.stream.core.query.RecordsAggregator.aggregate(RecordsAggregator.java:100) > at > org.apache.kylin.stream.core.query.StreamingCubeDataSearcher$StreamAggregateSearchResult.iterator(StreamingCubeDataSearcher.java:191) > at > org.apache.kylin.stream.server.rest.controller.DataController.query(DataController.java:119) > at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) > at > org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:848) > at > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:584) > at > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) > at > org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) > at org.eclipse.jetty.server.Server.handle(Server.java:534) > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333) > at > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) > at > org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) > at > org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) > at > org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) > at > org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) > at > org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) > at java.lang.Thread.run(Thread.java:748) > {code} > {code} > [root@cdh-worker-1 apache-kylin-3.0.0-SNAPSHOT-bin]# jstack 15512 | grep > org.apache.kylin.stream.core.query.MultiThreadsResultCollector | wc -l > 81 > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)