[
https://issues.apache.org/jira/browse/GRIFFIN-201?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16641223#comment-16641223
]
Nikolay Sokolov commented on GRIFFIN-201:
-----------------------------------------
Provided fix as part of https://github.com/apache/incubator-griffin/pull/431
> Hive metastore calls failing frequently
> ---------------------------------------
>
> Key: GRIFFIN-201
> URL: https://issues.apache.org/jira/browse/GRIFFIN-201
> Project: Griffin (Incubating)
> Issue Type: Bug
> Reporter: Nikolay Sokolov
> Priority: Major
>
> Observed both with vanilla docker image, current trunk in docker and on
> real-world deployments.
> Typical symptoms:
> * exception: org.apache.thrift.TApplicationException get_all_tables failed:
> out of sequence response
> * Got exception: org.apache.thrift.transport.TTransportException
> java.io.IOException: Stream closed
> * o.a.t.t.TIOStreamTransport : Error closing output stream.
> {code:none}
> 2018-10-07 14:35:13.797 ERROR 6148 --- [nio-8080-exec-1] h.log
> : Got exception: org.apache.thrift.TApplicationException
> get_all_tables failed: out of sequence response
> org.apache.thrift.TApplicationException: get_all_tables failed: out of
> sequence response
> at
> org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:84)
> ~[libthrift-0.9.3.jar!/:0.9.3]
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_tables(ThriftHiveMetastore.java:1443)
> ~[hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_tables(ThriftHiveMetastore.java:1430)
> ~[hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllTables(HiveMetaStoreClient.java:1425)
> [hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:87)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
> [spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> {code}
> {code:none}
> 2018-10-07 14:35:13.848 ERROR 6148 --- [nio-8080-exec-3] h.log
> : Got exception:
> org.apache.thrift.transport.TTransportException java.io.IOException: Stream
> closed
> org.apache.thrift.transport.TTransportException: java.io.IOException: Stream
> closed
> at
> org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:129)
> ~[libthrift-0.9.3.jar!/:0.9.3]
> at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86)
> ~[libthrift-0.9.3.jar!/:0.9.3]
> at
> org.apache.thrift.protocol.TBinaryProtocol.readStringBody(TBinaryProtocol.java:380)
> ~[libthrift-0.9.3.jar!/:0.9.3]
> at
> org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:230)
> ~[libthrift-0.9.3.jar!/:0.9.3]
> at
> org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77)
> ~[libthrift-0.9.3.jar!/:0.9.3]
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_tables(ThriftHiveMetastore.java:1443)
> ~[hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_tables(ThriftHiveMetastore.java:1430)
> ~[hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllTables(HiveMetaStoreClient.java:1425)
> [hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:87)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
> [spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> {code}
> {code:none}
> 2018-10-07 14:35:13.850 WARN 6148 --- [nio-8080-exec-1]
> o.a.t.t.TIOStreamTransport : Error closing output stream.
> java.net.SocketException: Socket closed
> at
> java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:118)
> ~[?:1.8.0_131]
> at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
> ~[?:1.8.0_131]
> at
> java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
> ~[?:1.8.0_131]
> at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
> ~[?:1.8.0_131]
> at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
> ~[?:1.8.0_131]
> at
> org.apache.thrift.transport.TIOStreamTransport.close(TIOStreamTransport.java:110)
> [libthrift-0.9.3.jar!/:0.9.3]
> at org.apache.thrift.transport.TSocket.close(TSocket.java:235)
> [libthrift-0.9.3.jar!/:0.9.3]
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.close(HiveMetaStoreClient.java:559)
> [hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.reconnect(HiveMetaStoreClient.java:374)
> [hive-metastore-2.2.0.jar!/:2.2.0]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.reconnect(HiveMetaStoreServiceImpl.java:193)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:89)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
> [spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61)
> [spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656)
> [spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> at
> org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47)
> [classes!/:0.3.1-incubating-SNAPSHOT]
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
