[
https://issues.apache.org/jira/browse/HBASE-18541?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16118851#comment-16118851
]
Enis Soztutar commented on HBASE-18541:
---------------------------------------
Just FYI, when I installed openjdk-8-dbg, I get this stack trace:
{code}
#0 0x00007fc70d8a3d18 in
ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState)
(thread=0x17e7000)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/os.hpp:419
#1 0x00007fc70d8a3d18 in
ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState)
(thread=0x17e7000)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/os/linux/vm/interfaceSupport_linux.hpp:31
#2 0x00007fc70d8a3d18 in
ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState)
(thread=thread@entry=0x17e7000, to=_thread_in_vm)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/interfaceSupport.hpp:212
#3 0x00007fc70d8a5346 in JVM_IHashCode(JNIEnv*, jobject) (to=_thread_in_vm,
this=<synthetic pointer>)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/interfaceSupport.hpp:231
#4 0x00007fc70d8a5346 in JVM_IHashCode(JNIEnv*, jobject) (thread=<optimized
out>, this=<synthetic pointer>)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/interfaceSupport.hpp:278
#5 0x00007fc70d8a5346 in JVM_IHashCode(JNIEnv*, jobject) (env=<optimized out>,
handle=0x7ffd1ded8970) at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/prims/jvm.cpp:542
#6 0x00007fc6fa61ebbf in [native offset=0xbf]
java.lang.System.identityHashCode(java.lang.Object) () at java/lang/System.java
#7 0x00007fc6fa27ba40 in [interpreted: bc = 4]
org.apache.hadoop.security.UserGroupInformation.hashCode() () at
org/apache/hadoop/security/UserGroupInformation.java:1616
#8 0x00007fc6fa27ba40 in [interpreted: bc = 4]
org.apache.hadoop.hbase.security.User.hashCode() () at
org/apache/hadoop/hbase/security/User.java:152
#9 0x00007fc6fa27ba40 in [interpreted: bc = 22]
org.apache.hadoop.hbase.ipc.ConnectionId.hashCode(org.apache.hadoop.hbase.security.User,java.lang.String,java.net.InetSocketAddress)
()
at org/apache/hadoop/hbase/ipc/ConnectionId.java:79
#10 0x00007fc6fa27ba40 in [interpreted: bc = 84]
org.apache.hadoop.hbase.ipc.AsyncRpcClient.createRpcChannel(java.lang.String,java.net.InetSocketAddress,org.apache.hadoop.hbase.security.User)
()
at org/apache/hadoop/hbase/ipc/AsyncRpcClient.java:413
#11 0x00007fc6fa27bd80 in [interpreted: bc = 24]
org.apache.hadoop.hbase.ipc.AsyncRpcClient.call(org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController,com.google.protobuf.Descriptors$MethodDescriptor,com.google.protobuf.Message,com.google.protobuf.Message,org.apache.hadoop.hbase.security.User,java.net.InetSocketAddress,org.apache.hadoop.hbase.client.MetricsConnection$CallStats)
()
at org/apache/hadoop/hbase/ipc/AsyncRpcClient.java:243
#12 0x00007fc6fa27bd80 in [interpreted: bc = 37]
org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(com.google.protobuf.Descriptors$MethodDescriptor,org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController,com.google.protobuf.Message,com.google.protobuf.Message,org.apache.hadoop.hbase.security.User,java.net.InetSocketAddress)
() at org/apache/hadoop/hbase/ipc/AbstractRpcClient.java:233
#13 0x00007fc6fa27bd80 in [interpreted: bc = 28]
org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(com.google.protobuf.Descriptors$MethodDescriptor,com.google.protobuf.RpcController,com.google.protobuf.Message,com.google.protobuf.Message)
() at org/apache/hadoop/hbase/ipc/AbstractRpcClient.java:354
#14 0x00007fc6fa27be54 in [interpreted: bc = 24]
org.apache.hadoop.hbase.protobuf.generated.MasterProtos$MasterService$BlockingStub.isMasterRunning(com.google.protobuf.RpcController,org.apache.hadoop.hbase.protobuf.generated.MasterProtos$IsMasterRunningRequest)
() at org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java:64354
#15 0x00007fc6fa27be54 in [interpreted: bc = 8]
org.apache.hadoop.hbase.client.ConnectionImplementation$MasterServiceState.isMasterRunning()
()
at org/apache/hadoop/hbase/client/ConnectionImplementation.java:939
#16 0x00007fc6fa27b7d0 in [interpreted: bc = 10]
org.apache.hadoop.hbase.client.ConnectionImplementation.isKeepAliveMasterConnectedAndRunning(org.apache.hadoop.hbase.client.ConnectionImplementation$MasterServiceState)
() at org/apache/hadoop/hbase/client/ConnectionImplementation.java:1699
#17 0x00007fc6fa27b7d0 in [interpreted: bc = 12]
org.apache.hadoop.hbase.client.ConnectionImplementation.getKeepAliveMasterService()
()
at org/apache/hadoop/hbase/client/ConnectionImplementation.java:1287
#18 0x00007fc6fa27be54 in [interpreted: bc = 5]
org.apache.hadoop.hbase.client.MasterCallable.prepare(boolean) () at
org/apache/hadoop/hbase/client/MasterCallable.java:39
#19 0x00007fc6fa27c042 in [interpreted: bc = 36]
org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(org.apache.hadoop.hbase.client.RetryingCallable,int)
()
at org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java:101
#20 0x00007fc6fa27be54 in [interpreted: bc = 11]
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(org.apache.hadoop.hbase.client.RetryingCallable,org.apache.hadoop.hbase.client.RpcRetryingCallerFactory,int,int)
() at org/apache/hadoop/hbase/client/HBaseAdmin.java:2891
#21 0x00007fc6fa27bd80 in [interpreted: bc = 13]
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(org.apache.hadoop.hbase.client.RetryingCallable)
()
at org/apache/hadoop/hbase/client/HBaseAdmin.java:2881
#22 0x00007fc6fa27bd80 in [interpreted: bc = 2]
org.apache.hadoop.hbase.client.HBaseAdmin.access$900(org.apache.hadoop.hbase.client.HBaseAdmin,org.apache.hadoop.hbase.client.RetryingCallable)
()
at org/apache/hadoop/hbase/client/HBaseAdmin.java:201
#23 0x00007fc6fa27bd80 in [interpreted: bc = 20]
org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(org.apache.hadoop.hbase.protobuf.generated.MasterProtos$GetProcedureResultRequest)
() at org/apache/hadoop/hbase/client/HBaseAdmin.java:3296
#24 0x00007fc6fa27bd80 in [interpreted: bc = 32]
org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(long,long)
() at org/apache/hadoop/hbase/client/HBaseAdmin.java:3265
#25 0x00007fc6fa27bd80 in [interpreted: bc = 36]
org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(long,java.util.concurrent.TimeUnit)
()
at org/apache/hadoop/hbase/client/HBaseAdmin.java:3207
#26 0x00007fc6fa27be54 in [interpreted: bc = 3]
org.apache.hadoop.hbase.client.HBaseAdmin.get(java.util.concurrent.Future,long,java.util.concurrent.TimeUnit)
()
at org/apache/hadoop/hbase/client/HBaseAdmin.java:1869
#27 0x00007fc6fa27bd80 in [interpreted: bc = 14]
org.apache.hadoop.hbase.client.HBaseAdmin.createTable(org.apache.hadoop.hbase.HTableDescriptor,byte)
()
at org/apache/hadoop/hbase/client/HBaseAdmin.java:475
#28 0x00007fc6fa27c042 in [interpreted: bc = 64]
org.apache.hadoop.hbase.HBaseTestingUtility.createTable(org.apache.hadoop.hbase.HTableDescriptor,byte,byte,org.apache.hadoop.conf.Configuration)
()
at org/apache/hadoop/hbase/HBaseTestingUtility.java:1465
#29 0x00007fc6fa27bd80 in [interpreted: bc = 13]
org.apache.hadoop.hbase.HBaseTestingUtility.createTable(org.apache.hadoop.hbase.TableName,byte,byte,org.apache.hadoop.conf.Configuration)
()
at org/apache/hadoop/hbase/HBaseTestingUtility.java:1496
#30 0x00007fc6fa27bd80 in [interpreted: bc = 15]
org.apache.hadoop.hbase.HBaseTestingUtility.createTable(org.apache.hadoop.hbase.TableName,byte,byte)
()
at org/apache/hadoop/hbase/HBaseTestingUtility.java:1411
#31 0x00007fc6fa2744e7 in StubRoutines (1) ()
#32 0x00007fc70d86315b in JavaCalls::call_helper(JavaValue*, methodHandle*,
JavaCallArguments*, Thread*) (result=0x7ffd1ded98c0, m=<optimized out>,
args=<optimized out>, __the_thread__=<optimized out>)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/javaCalls.cpp:406
#33 0x00007fc70d87ceba in jni_invoke_nonstatic(JNIEnv*, JavaValue*, jobject,
jmethodID, JNI_ArgumentPusher*, Thread*, JNICallType) (env=env@entry=0x17e71e0,
result=result@entry=0x7ffd1ded98c0, receiver=receiver@entry=0x184b240,
method_id=<optimized out>, args=args@entry=0x7ffd1ded98f0,
__the_thread__=__the_thread__@entry=0x17e7000, call_type=JNI_VIRTUAL)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/prims/jni.cpp:1397
#34 0x00007fc70d87eb85 in jni_CallObjectMethodV(JNIEnv *, jobject, jmethodID,
typedef __va_list_tag __va_list_tag *) (env=0x17e71e0, obj=0x184b240,
methodID=<optimized out>, args=0x7ffd1ded9a30)
at
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/prims/jni.cpp:1729
#35 0x0000000000439d18 in JNIEnv_::CallObjectMethod(_jobject*, _jmethodID*,
...) (this=<optimized out>, obj=<optimized out>, methodID=<optimized out>)
at /usr/lib/jvm/java-8-openjdk-amd64//include/jni.h:901
#36 0x00000000004392b8 in hbase::MiniCluster::CreateTable(std::string const&,
std::vector<std::string, std::allocator<std::string> > const&,
std::vector<std::string, std::allocator<std::string> > const&) (this=<optimized
out>, table=..., families=..., keys=std::vector of length -17561613817511,
capacity -17561613817516 = {...}) at test-util/mini-cluster.cc:264
#37 0x0000000000438ec9 in hbase::MiniCluster::CreateTable(std::string const&,
std::string const&, std::vector<std::string, std::allocator<std::string> >
const&) (this=0x17d0000, table="split-table9", family=..., keys=std::vector of
length 10, capacity 10 = {...}) at test-util/mini-cluster.cc:240
#38 0x000000000041b84a in runMultiTest(std::shared_ptr<AsyncRegionLocatorBase>,
std::string const&, bool, unsigned int, unsigned int, unsigned int)
(region_locator=warning: RTTI symbol not found for class
'std::_Sp_counted_ptr_inplace<MockWrongRegionAsyncRegionLocator,
std::allocator<MockWrongRegionAsyncRegionLocator>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class
'std::_Sp_counted_ptr_inplace<MockWrongRegionAsyncRegionLocator,
std::allocator<MockWrongRegionAsyncRegionLocator>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr (count 2, weak 0) 0x49e1f90, table_name=...,
split_regions=<optimized out>, tries=<optimized out>,
operation_timeout_millis=600000, num_rows=10000)
at core/async-batch-rpc-retrying-test.h:287
#39 0x0000000000420271 in
AsyncBatchRpcRetryTest_FailWithExceptionSplitRegions_Test::TestBody()
(this=<optimized out>) at core/async-batch-rpc-retrying-multi-region-test.cc:40
#40 0x00000000005900de in
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
(method=<optimized out>, location=<optimized out>, object=<optimized out>) at
third-party/googletest/googletest/src/gtest.cc:2402
#41 0x00000000005900de in
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
(object=<optimized out>, method=<optimized out>, location=0x8726c5 "the test
body") at third-party/googletest/googletest/src/gtest.cc:2438
#42 0x0000000000570114 in testing::Test::Run() (this=<optimized out>) at
third-party/googletest/googletest/src/gtest.cc:2474
#43 0x00000000005713c3 in testing::TestInfo::Run() (this=<optimized out>) at
third-party/googletest/googletest/src/gtest.cc:2656
#44 0x0000000000571af3 in testing::TestCase::Run() (this=<optimized out>) at
third-party/googletest/googletest/src/gtest.cc:2774
#45 0x000000000057c17b in testing::internal::UnitTestImpl::RunAllTests()
(this=<optimized out>) at third-party/googletest/googletest/src/gtest.cc:4649
#46 0x0000000000590c3c in
testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
bool>(testing::internal::UnitTestImpl*, bool
(testing::internal::UnitTestImpl::*)(), char const*) (method=<optimized out>,
location=<optimized out>, object=<optimized out>) at
third-party/googletest/googletest/src/gtest.cc:2402
#47 0x0000000000590c3c in
testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
bool>(testing::internal::UnitTestImpl*, bool
(testing::internal::UnitTestImpl::*)(), char const*) (object=<optimized out>,
method=<optimized out>, location=0x872db3 "auxiliary test code (environments or
event listeners)") at third-party/googletest/googletest/src/gtest.cc:2438
#48 0x000000000057bc1a in testing::UnitTest::Run() (this=0xbfe460
<testing::UnitTest::GetInstance()::instance>) at
third-party/googletest/googletest/src/gtest.cc:4257
#49 0x00000000005631ed in main(int, char**) () at
third-party/googletest/googletest/include/gtest/gtest.h:2233
#50 0x00000000005631ed in main(int, char**) (argc=<optimized out>,
argv=<optimized out>) at third-party/googletest/googlemock/src/gmock_main.cc:53
{code}
> [C++] Segfaults from JNI
> ------------------------
>
> Key: HBASE-18541
> URL: https://issues.apache.org/jira/browse/HBASE-18541
> Project: HBase
> Issue Type: Sub-task
> Reporter: Enis Soztutar
> Assignee: Ted Yu
>
> retry-test and multi-retry-test fails flakily when run with
> {code}
> buck test --all --no-results-cache
> {code}
> or when run in a loop:
> {code}
> for i in `seq 1 10`; do buck test --no-results-cache core:retry-test || break
> 1; done
> {code}
> The problem seems to be within the JNI internals and usually happens at the
> create table method call. I was not able to inspect much, but the comments in
> our mini-cluster indicate that we may need to use global references instead
> of local ones. I suspect the problem happens when there is a GC run for the
> test since the failure happens usually after some time (but almost always in
> create table method).
> [~ted_yu] do you mind taking a look at this.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
