[ 
https://issues.apache.org/jira/browse/HBASE-18541?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16118851#comment-16118851
 ] 

Enis Soztutar commented on HBASE-18541:
---------------------------------------

Just FYI, when I installed openjdk-8-dbg, I get this stack trace: 
{code}
#0  0x00007fc70d8a3d18 in 
ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState) 
(thread=0x17e7000)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/os.hpp:419
#1  0x00007fc70d8a3d18 in 
ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState) 
(thread=0x17e7000)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/os/linux/vm/interfaceSupport_linux.hpp:31
#2  0x00007fc70d8a3d18 in 
ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState) 
(thread=thread@entry=0x17e7000, to=_thread_in_vm)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/interfaceSupport.hpp:212
#3  0x00007fc70d8a5346 in JVM_IHashCode(JNIEnv*, jobject) (to=_thread_in_vm, 
this=<synthetic pointer>)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/interfaceSupport.hpp:231
#4  0x00007fc70d8a5346 in JVM_IHashCode(JNIEnv*, jobject) (thread=<optimized 
out>, this=<synthetic pointer>)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/interfaceSupport.hpp:278
#5  0x00007fc70d8a5346 in JVM_IHashCode(JNIEnv*, jobject) (env=<optimized out>, 
handle=0x7ffd1ded8970) at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/prims/jvm.cpp:542
#6  0x00007fc6fa61ebbf in [native offset=0xbf] 
java.lang.System.identityHashCode(java.lang.Object) () at java/lang/System.java
#7  0x00007fc6fa27ba40 in [interpreted: bc = 4] 
org.apache.hadoop.security.UserGroupInformation.hashCode() () at 
org/apache/hadoop/security/UserGroupInformation.java:1616
#8  0x00007fc6fa27ba40 in [interpreted: bc = 4] 
org.apache.hadoop.hbase.security.User.hashCode() () at 
org/apache/hadoop/hbase/security/User.java:152
#9  0x00007fc6fa27ba40 in [interpreted: bc = 22] 
org.apache.hadoop.hbase.ipc.ConnectionId.hashCode(org.apache.hadoop.hbase.security.User,java.lang.String,java.net.InetSocketAddress)
 ()
    at org/apache/hadoop/hbase/ipc/ConnectionId.java:79
#10 0x00007fc6fa27ba40 in [interpreted: bc = 84] 
org.apache.hadoop.hbase.ipc.AsyncRpcClient.createRpcChannel(java.lang.String,java.net.InetSocketAddress,org.apache.hadoop.hbase.security.User)
 ()
    at org/apache/hadoop/hbase/ipc/AsyncRpcClient.java:413
#11 0x00007fc6fa27bd80 in [interpreted: bc = 24] 
org.apache.hadoop.hbase.ipc.AsyncRpcClient.call(org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController,com.google.protobuf.Descriptors$MethodDescriptor,com.google.protobuf.Message,com.google.protobuf.Message,org.apache.hadoop.hbase.security.User,java.net.InetSocketAddress,org.apache.hadoop.hbase.client.MetricsConnection$CallStats)
 ()
    at org/apache/hadoop/hbase/ipc/AsyncRpcClient.java:243
#12 0x00007fc6fa27bd80 in [interpreted: bc = 37] 
org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(com.google.protobuf.Descriptors$MethodDescriptor,org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController,com.google.protobuf.Message,com.google.protobuf.Message,org.apache.hadoop.hbase.security.User,java.net.InetSocketAddress)
 () at org/apache/hadoop/hbase/ipc/AbstractRpcClient.java:233
#13 0x00007fc6fa27bd80 in [interpreted: bc = 28] 
org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(com.google.protobuf.Descriptors$MethodDescriptor,com.google.protobuf.RpcController,com.google.protobuf.Message,com.google.protobuf.Message)
 () at org/apache/hadoop/hbase/ipc/AbstractRpcClient.java:354
#14 0x00007fc6fa27be54 in [interpreted: bc = 24] 
org.apache.hadoop.hbase.protobuf.generated.MasterProtos$MasterService$BlockingStub.isMasterRunning(com.google.protobuf.RpcController,org.apache.hadoop.hbase.protobuf.generated.MasterProtos$IsMasterRunningRequest)
 () at org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java:64354
#15 0x00007fc6fa27be54 in [interpreted: bc = 8] 
org.apache.hadoop.hbase.client.ConnectionImplementation$MasterServiceState.isMasterRunning()
 ()
    at org/apache/hadoop/hbase/client/ConnectionImplementation.java:939
#16 0x00007fc6fa27b7d0 in [interpreted: bc = 10] 
org.apache.hadoop.hbase.client.ConnectionImplementation.isKeepAliveMasterConnectedAndRunning(org.apache.hadoop.hbase.client.ConnectionImplementation$MasterServiceState)
 () at org/apache/hadoop/hbase/client/ConnectionImplementation.java:1699
#17 0x00007fc6fa27b7d0 in [interpreted: bc = 12] 
org.apache.hadoop.hbase.client.ConnectionImplementation.getKeepAliveMasterService()
 ()
    at org/apache/hadoop/hbase/client/ConnectionImplementation.java:1287
#18 0x00007fc6fa27be54 in [interpreted: bc = 5] 
org.apache.hadoop.hbase.client.MasterCallable.prepare(boolean) () at 
org/apache/hadoop/hbase/client/MasterCallable.java:39
#19 0x00007fc6fa27c042 in [interpreted: bc = 36] 
org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(org.apache.hadoop.hbase.client.RetryingCallable,int)
 ()
    at org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java:101
#20 0x00007fc6fa27be54 in [interpreted: bc = 11] 
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(org.apache.hadoop.hbase.client.RetryingCallable,org.apache.hadoop.hbase.client.RpcRetryingCallerFactory,int,int)
 () at org/apache/hadoop/hbase/client/HBaseAdmin.java:2891
#21 0x00007fc6fa27bd80 in [interpreted: bc = 13] 
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(org.apache.hadoop.hbase.client.RetryingCallable)
 ()
    at org/apache/hadoop/hbase/client/HBaseAdmin.java:2881
#22 0x00007fc6fa27bd80 in [interpreted: bc = 2] 
org.apache.hadoop.hbase.client.HBaseAdmin.access$900(org.apache.hadoop.hbase.client.HBaseAdmin,org.apache.hadoop.hbase.client.RetryingCallable)
 ()
    at org/apache/hadoop/hbase/client/HBaseAdmin.java:201
#23 0x00007fc6fa27bd80 in [interpreted: bc = 20] 
org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(org.apache.hadoop.hbase.protobuf.generated.MasterProtos$GetProcedureResultRequest)
 () at org/apache/hadoop/hbase/client/HBaseAdmin.java:3296
#24 0x00007fc6fa27bd80 in [interpreted: bc = 32] 
org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(long,long)
 () at org/apache/hadoop/hbase/client/HBaseAdmin.java:3265
#25 0x00007fc6fa27bd80 in [interpreted: bc = 36] 
org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(long,java.util.concurrent.TimeUnit)
 ()
    at org/apache/hadoop/hbase/client/HBaseAdmin.java:3207
#26 0x00007fc6fa27be54 in [interpreted: bc = 3] 
org.apache.hadoop.hbase.client.HBaseAdmin.get(java.util.concurrent.Future,long,java.util.concurrent.TimeUnit)
 ()
    at org/apache/hadoop/hbase/client/HBaseAdmin.java:1869
#27 0x00007fc6fa27bd80 in [interpreted: bc = 14] 
org.apache.hadoop.hbase.client.HBaseAdmin.createTable(org.apache.hadoop.hbase.HTableDescriptor,byte)
 ()
    at org/apache/hadoop/hbase/client/HBaseAdmin.java:475
#28 0x00007fc6fa27c042 in [interpreted: bc = 64] 
org.apache.hadoop.hbase.HBaseTestingUtility.createTable(org.apache.hadoop.hbase.HTableDescriptor,byte,byte,org.apache.hadoop.conf.Configuration)
 ()
    at org/apache/hadoop/hbase/HBaseTestingUtility.java:1465
#29 0x00007fc6fa27bd80 in [interpreted: bc = 13] 
org.apache.hadoop.hbase.HBaseTestingUtility.createTable(org.apache.hadoop.hbase.TableName,byte,byte,org.apache.hadoop.conf.Configuration)
 ()
    at org/apache/hadoop/hbase/HBaseTestingUtility.java:1496
#30 0x00007fc6fa27bd80 in [interpreted: bc = 15] 
org.apache.hadoop.hbase.HBaseTestingUtility.createTable(org.apache.hadoop.hbase.TableName,byte,byte)
 ()
    at org/apache/hadoop/hbase/HBaseTestingUtility.java:1411
#31 0x00007fc6fa2744e7 in StubRoutines (1) ()
#32 0x00007fc70d86315b in JavaCalls::call_helper(JavaValue*, methodHandle*, 
JavaCallArguments*, Thread*) (result=0x7ffd1ded98c0, m=<optimized out>, 
args=<optimized out>, __the_thread__=<optimized out>)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/runtime/javaCalls.cpp:406
#33 0x00007fc70d87ceba in jni_invoke_nonstatic(JNIEnv*, JavaValue*, jobject, 
jmethodID, JNI_ArgumentPusher*, Thread*, JNICallType) (env=env@entry=0x17e71e0, 
result=result@entry=0x7ffd1ded98c0, receiver=receiver@entry=0x184b240, 
method_id=<optimized out>, args=args@entry=0x7ffd1ded98f0, 
__the_thread__=__the_thread__@entry=0x17e7000, call_type=JNI_VIRTUAL)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/prims/jni.cpp:1397
#34 0x00007fc70d87eb85 in jni_CallObjectMethodV(JNIEnv *, jobject, jmethodID, 
typedef __va_list_tag __va_list_tag *) (env=0x17e71e0, obj=0x184b240, 
methodID=<optimized out>, args=0x7ffd1ded9a30)
    at 
/build/openjdk-8-pZyJp3/openjdk-8-8u131-b11/src/hotspot/src/share/vm/prims/jni.cpp:1729
#35 0x0000000000439d18 in JNIEnv_::CallObjectMethod(_jobject*, _jmethodID*, 
...) (this=<optimized out>, obj=<optimized out>, methodID=<optimized out>)
    at /usr/lib/jvm/java-8-openjdk-amd64//include/jni.h:901
#36 0x00000000004392b8 in hbase::MiniCluster::CreateTable(std::string const&, 
std::vector<std::string, std::allocator<std::string> > const&, 
std::vector<std::string, std::allocator<std::string> > const&) (this=<optimized 
out>, table=..., families=..., keys=std::vector of length -17561613817511, 
capacity -17561613817516 = {...}) at test-util/mini-cluster.cc:264
#37 0x0000000000438ec9 in hbase::MiniCluster::CreateTable(std::string const&, 
std::string const&, std::vector<std::string, std::allocator<std::string> > 
const&) (this=0x17d0000, table="split-table9", family=..., keys=std::vector of 
length 10, capacity 10 = {...}) at test-util/mini-cluster.cc:240
#38 0x000000000041b84a in runMultiTest(std::shared_ptr<AsyncRegionLocatorBase>, 
std::string const&, bool, unsigned int, unsigned int, unsigned int) 
(region_locator=warning: RTTI symbol not found for class 
'std::_Sp_counted_ptr_inplace<MockWrongRegionAsyncRegionLocator, 
std::allocator<MockWrongRegionAsyncRegionLocator>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 
'std::_Sp_counted_ptr_inplace<MockWrongRegionAsyncRegionLocator, 
std::allocator<MockWrongRegionAsyncRegionLocator>, (__gnu_cxx::_Lock_policy)2>'

std::shared_ptr (count 2, weak 0) 0x49e1f90, table_name=..., 
split_regions=<optimized out>, tries=<optimized out>, 
operation_timeout_millis=600000, num_rows=10000)
    at core/async-batch-rpc-retrying-test.h:287
#39 0x0000000000420271 in 
AsyncBatchRpcRetryTest_FailWithExceptionSplitRegions_Test::TestBody() 
(this=<optimized out>) at core/async-batch-rpc-retrying-multi-region-test.cc:40
#40 0x00000000005900de in 
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, 
void>(testing::Test*, void (testing::Test::*)(), char const*) 
(method=<optimized out>, location=<optimized out>, object=<optimized out>) at 
third-party/googletest/googletest/src/gtest.cc:2402
#41 0x00000000005900de in 
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, 
void>(testing::Test*, void (testing::Test::*)(), char const*) 
(object=<optimized out>, method=<optimized out>, location=0x8726c5 "the test 
body") at third-party/googletest/googletest/src/gtest.cc:2438
#42 0x0000000000570114 in testing::Test::Run() (this=<optimized out>) at 
third-party/googletest/googletest/src/gtest.cc:2474
#43 0x00000000005713c3 in testing::TestInfo::Run() (this=<optimized out>) at 
third-party/googletest/googletest/src/gtest.cc:2656
#44 0x0000000000571af3 in testing::TestCase::Run() (this=<optimized out>) at 
third-party/googletest/googletest/src/gtest.cc:2774
#45 0x000000000057c17b in testing::internal::UnitTestImpl::RunAllTests() 
(this=<optimized out>) at third-party/googletest/googletest/src/gtest.cc:4649
#46 0x0000000000590c3c in 
testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
 bool>(testing::internal::UnitTestImpl*, bool 
(testing::internal::UnitTestImpl::*)(), char const*) (method=<optimized out>, 
location=<optimized out>, object=<optimized out>) at 
third-party/googletest/googletest/src/gtest.cc:2402
#47 0x0000000000590c3c in 
testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
 bool>(testing::internal::UnitTestImpl*, bool 
(testing::internal::UnitTestImpl::*)(), char const*) (object=<optimized out>, 
method=<optimized out>, location=0x872db3 "auxiliary test code (environments or 
event listeners)") at third-party/googletest/googletest/src/gtest.cc:2438
#48 0x000000000057bc1a in testing::UnitTest::Run() (this=0xbfe460 
<testing::UnitTest::GetInstance()::instance>) at 
third-party/googletest/googletest/src/gtest.cc:4257
#49 0x00000000005631ed in main(int, char**) () at 
third-party/googletest/googletest/include/gtest/gtest.h:2233
#50 0x00000000005631ed in main(int, char**) (argc=<optimized out>, 
argv=<optimized out>) at third-party/googletest/googlemock/src/gmock_main.cc:53
{code}

> [C++] Segfaults from JNI
> ------------------------
>
>                 Key: HBASE-18541
>                 URL: https://issues.apache.org/jira/browse/HBASE-18541
>             Project: HBase
>          Issue Type: Sub-task
>            Reporter: Enis Soztutar
>            Assignee: Ted Yu
>
> retry-test and multi-retry-test fails flakily when run with 
> {code}
> buck test --all --no-results-cache
> {code}
> or when run in a loop:
> {code}
> for i in `seq 1 10`; do buck test --no-results-cache core:retry-test || break 
> 1; done
> {code}
> The problem seems to be within the JNI internals and usually happens at the 
> create table method call. I was not able to inspect much, but the comments in 
> our mini-cluster indicate that we may need to use global references instead 
> of local ones. I suspect the problem happens when there is a GC run for the 
> test since the failure happens usually after some time (but almost always in 
> create table method). 
> [~ted_yu] do you mind taking a look at this. 



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Reply via email to