Nikolay Sokolov created GRIFFIN-201:
---------------------------------------
Summary: Hive metastore calls failing frequently
Key: GRIFFIN-201
URL: https://issues.apache.org/jira/browse/GRIFFIN-201
Project: Griffin (Incubating)
Issue Type: Bug
Reporter: Nikolay Sokolov
Observed both with vanilla docker image, current trunk in docker and on
real-world deployments.
Typical symptoms:
* exception: org.apache.thrift.TApplicationException get_all_tables failed: out
of sequence response
* Got exception: org.apache.thrift.transport.TTransportException
java.io.IOException: Stream closed
* o.a.t.t.TIOStreamTransport : Error closing output stream.
{code:none}
2018-10-07 14:35:13.797 ERROR 6148 --- [nio-8080-exec-1] h.log
: Got exception: org.apache.thrift.TApplicationException
get_all_tables failed: out of sequence response
org.apache.thrift.TApplicationException: get_all_tables failed: out of sequence
response
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:84)
~[libthrift-0.9.3.jar!/:0.9.3]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_tables(ThriftHiveMetastore.java:1443)
~[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_tables(ThriftHiveMetastore.java:1430)
~[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllTables(HiveMetaStoreClient.java:1425)
[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:87)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
[spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47)
[classes!/:0.3.1-incubating-SNAPSHOT]
{code}
{code:none}
2018-10-07 14:35:13.848 ERROR 6148 --- [nio-8080-exec-3] h.log
: Got exception:
org.apache.thrift.transport.TTransportException java.io.IOException: Stream
closed
org.apache.thrift.transport.TTransportException: java.io.IOException: Stream
closed
at
org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:129)
~[libthrift-0.9.3.jar!/:0.9.3]
at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86)
~[libthrift-0.9.3.jar!/:0.9.3]
at
org.apache.thrift.protocol.TBinaryProtocol.readStringBody(TBinaryProtocol.java:380)
~[libthrift-0.9.3.jar!/:0.9.3]
at
org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:230)
~[libthrift-0.9.3.jar!/:0.9.3]
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77)
~[libthrift-0.9.3.jar!/:0.9.3]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_tables(ThriftHiveMetastore.java:1443)
~[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_tables(ThriftHiveMetastore.java:1430)
~[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllTables(HiveMetaStoreClient.java:1425)
[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:87)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
[spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47)
[classes!/:0.3.1-incubating-SNAPSHOT]
{code}
{code:none}
2018-10-07 14:35:13.850 WARN 6148 --- [nio-8080-exec-1]
o.a.t.t.TIOStreamTransport : Error closing output stream.
java.net.SocketException: Socket closed
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:118)
~[?:1.8.0_131]
at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
~[?:1.8.0_131]
at
java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
~[?:1.8.0_131]
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
~[?:1.8.0_131]
at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
~[?:1.8.0_131]
at
org.apache.thrift.transport.TIOStreamTransport.close(TIOStreamTransport.java:110)
[libthrift-0.9.3.jar!/:0.9.3]
at org.apache.thrift.transport.TSocket.close(TSocket.java:235)
[libthrift-0.9.3.jar!/:0.9.3]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.close(HiveMetaStoreClient.java:559)
[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.reconnect(HiveMetaStoreClient.java:374)
[hive-metastore-2.2.0.jar!/:2.2.0]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.reconnect(HiveMetaStoreServiceImpl.java:193)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl.getAllTableNames(HiveMetaStoreServiceImpl.java:89)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$FastClassBySpringCGLIB$$d0fbb087.invoke(<generated>)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
[spring-core-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheInterceptor$1.invoke(CacheInterceptor.java:52)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:345)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:408)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:327)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61)
[spring-context-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656)
[spring-aop-4.3.6.RELEASE.jar!/:4.3.6.RELEASE]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreServiceImpl$$EnhancerBySpringCGLIB$$390ab847.getAllTableNames(<generated>)
[classes!/:0.3.1-incubating-SNAPSHOT]
at
org.apache.griffin.core.metastore.hive.HiveMetaStoreController.getAllTableNames(HiveMetaStoreController.java:47)
[classes!/:0.3.1-incubating-SNAPSHOT]
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
