[ https://issues.apache.org/jira/browse/GRIFFIN-201?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Nikolay Sokolov closed GRIFFIN-201. ----------------------------------- Resolution: Fixed > Hive metastore calls failing frequently > --------------------------------------- > > Key: GRIFFIN-201 > URL: https://issues.apache.org/jira/browse/GRIFFIN-201 > Project: Griffin (Incubating) > Issue Type: Bug > Reporter: Nikolay Sokolov > Priority: Major > > Observed both with vanilla docker image, current trunk in docker and on > real-world deployments. > Typical symptoms: > * exception: org.apache.thrift.TApplicationException get_all_tables failed: > out of sequence response > * Got exception: org.apache.thrift.transport.TTransportException > java.io.IOException: Stream closed > * o.a.t.t.TIOStreamTransport : Error closing output stream. > {code:none} > 2018-10-07 14:35:13.797 ERROR 6148 --- [nio-8080-exec-1] h.log > : Got exception: org.apache.thrift.TApplicationException > get_all_tables failed: out of sequence response > org.apache.thrift.TApplicationException: get_all_tables failed: out of > sequence response > at > org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:84) > ~[libthrift-0.9.3.jar!/:0.9.3] > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_tables(ThriftHiveMetastore.java:1443) > ~[hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_tables(ThriftHiveMetastore.java:1430) > ~[hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllTables(HiveMetaStoreClient.java:1425) > [hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:87) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) > [spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47) > [classes!/:0.3.1-incubating-SNAPSHOT] > {code} > {code:none} > 2018-10-07 14:35:13.848 ERROR 6148 --- [nio-8080-exec-3] h.log > : Got exception: > org.apache.thrift.transport.TTransportException java.io.IOException: Stream > closed > org.apache.thrift.transport.TTransportException: java.io.IOException: Stream > closed > at > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:129) > ~[libthrift-0.9.3.jar!/:0.9.3] > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) > ~[libthrift-0.9.3.jar!/:0.9.3] > at > org.apache.thrift.protocol.TBinaryProtocol.readStringBody(TBinaryProtocol.java:380) > ~[libthrift-0.9.3.jar!/:0.9.3] > at > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:230) > ~[libthrift-0.9.3.jar!/:0.9.3] > at > org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77) > ~[libthrift-0.9.3.jar!/:0.9.3] > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_tables(ThriftHiveMetastore.java:1443) > ~[hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_tables(ThriftHiveMetastore.java:1430) > ~[hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllTables(HiveMetaStoreClient.java:1425) > [hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:87) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) > [spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47) > [classes!/:0.3.1-incubating-SNAPSHOT] > {code} > {code:none} > 2018-10-07 14:35:13.850 WARN 6148 --- [nio-8080-exec-1] > o.a.t.t.TIOStreamTransport : Error closing output stream. > java.net.SocketException: Socket closed > at > java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:118) > ~[?:1.8.0_131] > at java.net.SocketOutputStream.write(SocketOutputStream.java:155) > ~[?:1.8.0_131] > at > java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) > ~[?:1.8.0_131] > at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) > ~[?:1.8.0_131] > at java.io.FilterOutputStream.close(FilterOutputStream.java:158) > ~[?:1.8.0_131] > at > org.apache.thrift.transport.TIOStreamTransport.close(TIOStreamTransport.java:110) > [libthrift-0.9.3.jar!/:0.9.3] > at org.apache.thrift.transport.TSocket.close(TSocket.java:235) > [libthrift-0.9.3.jar!/:0.9.3] > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.close(HiveMetaStoreClient.java:559) > [hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.reconnect(HiveMetaStoreClient.java:374) > [hive-metastore-2.2.0.jar!/:2.2.0] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.reconnect(HiveMetaStoreServiceImpl.java:193) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:89) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) > [spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61) > [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) > [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>) > [classes!/:0.3.1-incubating-SNAPSHOT] > at > org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47) > [classes!/:0.3.1-incubating-SNAPSHOT] > {code} -- This message was sent by Atlassian JIRA (v7.6.3#76005)