[ 
https://issues.apache.org/jira/browse/CASSANDRA-14380?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Adam Geiger updated CASSANDRA-14380:
------------------------------------
    Description: 
Running Cassandra with a Rook Ceph filesystem within Kubernetes.  During the 
startup, the following Warnings in the debug log pop up and then Cassandra 
crashes shortly after and restarts.  It looks like before hitting this error, 
it is doing a lot of writing and flushing

WARN [MemtableFlushWriter:2] 2018-04-11 14:34:42,748 NativeLibrary.java:328 - 
fsync(666) failed, errorno (22) {}
com.sun.jna.LastErrorException: [22] Invalid argument
 at org.apache.cassandra.utils.NativeLibraryLinux.fsync(Native Method) 
~[apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.utils.NativeLibraryLinux.callFsync(NativeLibraryLinux.java:107)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
 at org.apache.cassandra.utils.NativeLibrary.trySync(NativeLibrary.java:317) 
~[apache-cassandra-3.11.0.jar:3.11.0]
 at org.apache.cassandra.utils.SyncUtil.trySync(SyncUtil.java:179) 
[apache-cassandra-3.11.0.jar:3.11.0]
 at org.apache.cassandra.utils.SyncUtil.trySyncDir(SyncUtil.java:190) 
[apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.util.SequentialWriter.openChannel(SequentialWriter.java:107)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.util.SequentialWriter.<init>(SequentialWriter.java:141) 
[apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.big.BigTableWriter.writeMetadata(BigTableWriter.java:402)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.big.BigTableWriter.access$300(BigTableWriter.java:53)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.big.BigTableWriter$TransactionalProxy.doPrepare(BigTableWriter.java:368)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.prepareToCommit(Transactional.java:173)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.SSTableWriter.prepareToCommit(SSTableWriter.java:281)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter.prepareToCommit(SimpleSSTableMultiWriter.java:101)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1153)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) 
[na:1.8.0]
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) 
[na:1.8.0]
 at 
org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000BCF32600.run(Unknown
 Source) ~[na:na]
 at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]

 
Syslog shows the following 
(logs-from-cassandra-in-r97bb66e967-apiconnect-cc-0.txt):

INFO  [main] 2018-04-11 14:49:01,848 ColumnFamilyStore.java:406 - Initializing 
apim.ur_to_op_by_op
INFO  [MemoryMXBean notification dispatcher] 2018-04-11 14:49:25,889 
GCInspector.java:284 - global GC in 206ms.  class storage: 28700680 -> 
28692744; miscellaneous non-heap storage: 49871216 -> 53570176; 
nursery-allocate: 1296878920 -> 149116672; tenured-SOA: 140321968 -> 139143760
#0: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x302a94) 
[0x7f17e4f10a94]
#1: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x306b2d) 
[0x7f17e4f14b2d]
#2: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xc82da) 
[0x7f17e4cd62da]
#3: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x22056) 
[0x7f17e6531056]
#4: /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390) [0x7f17ed0de390]
#5: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2c4e1f) 
[0x7f17e4ed2e1f]
#6: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x158c04) 
[0x7f17e4d66c04]
#7: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542d24) 
[0x7f17e5150d24]
#8: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542e0b) 
[0x7f17e5150e0b]
#9: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x54981a) 
[0x7f17e515781a]
#10: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x5494c8) 
[0x7f17e51574c8]
#11: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x548dd2) 
[0x7f17e5156dd2]
#12: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2d7019) 
[0x7f17e4ee5019]
#13: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd31ee) 
[0x7f17e4ce11ee]
#14: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd3e51) 
[0x7f17e4ce1e51]
#15: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#16: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd5ce5) 
[0x7f17e4ce3ce5]
#17: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd668b) 
[0x7f17e4ce468b]
#18: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6d20) 
[0x7f17e4ce4d20]
#19: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6fea) 
[0x7f17e4ce4fea]
#20: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd70b4) 
[0x7f17e4ce50b4]
#21: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#22: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd743f) 
[0x7f17e4ce543f]
#23: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9thr29.so(+0xa38e) 
[0x7f17e699738e]
#24: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f17ed0d46ba]
#25: function clone+0x6d [0x7f17efdd382d]
Unhandled exception
Type=Segmentation error vmState=0x0005ff01
J9Generic_Signal_Number=00000004 Signal_Number=0000000b Error_Value=00000000 
Signal_Code=00000001
Handler1=00007F17E6C26460 Handler2=00007F17E6530E40 
InaccessibleAddress=000000000000001A
RDI=00007F17B50070B0 RSI=00007F17C8E51100 RAX=0000000000000012 
RBX=00007F17B5007120
RCX=0000000000000000 RDX=00007F17E577BC90 R8=00007F17B5070670 
R9=0000000000000000
R10=0000000000000011 R11=00007F17E5459071 R12=00007F17C8E536D8 
R13=00007F17B5006420
R14=00007F17B5006290 R15=00007F17C4016CB0
RIP=00007F17E4ED2E1F GS=0000 FS=0000 RSP=00007F17C8E51150
EFlags=0000000000010216 CS=E033 RBP=00007F17B5000000 ERR=0000000000000004
TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=000000000000001A
xmm0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm1 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm4 00000000000000ff (f: 255.000000, d: 1.259867e-321)
xmm5 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
Module=/opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so
Module_base_address=00007F17E4C0E000
Method_being_compiled=org/apache/cassandra/utils/concurrent/Locks.monitorExitUnsafe(Ljava/lang/Object;)V
Target=2_90_20171215_373586 (Linux 4.4.0-116-generic)
CPU=amd64 (8 logical CPUs) (0xbc2d25000 RAM)
----------- Stack Backtrace -----------
(0x00007F17E4ED2E1F [libj9jit29.so+0x2c4e1f])
(0x00007F17E4D66C04 [libj9jit29.so+0x158c04])
(0x00007F17E5150D24 [libj9jit29.so+0x542d24])
(0x00007F17E5150E0B [libj9jit29.so+0x542e0b])
(0x00007F17E515781A [libj9jit29.so+0x54981a])
(0x00007F17E51574C8 [libj9jit29.so+0x5494c8])
(0x00007F17E5156DD2 [libj9jit29.so+0x548dd2])
(0x00007F17E4EE5019 [libj9jit29.so+0x2d7019])
(0x00007F17E4CE11EE [libj9jit29.so+0xd31ee])
(0x00007F17E4CE1E51 [libj9jit29.so+0xd3e51])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE3CE5 [libj9jit29.so+0xd5ce5])
(0x00007F17E4CE468B [libj9jit29.so+0xd668b])
(0x00007F17E4CE4D20 [libj9jit29.so+0xd6d20])
(0x00007F17E4CE4FEA [libj9jit29.so+0xd6fea])
(0x00007F17E4CE50B4 [libj9jit29.so+0xd70b4])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE543F [libj9jit29.so+0xd743f])
(0x00007F17E699738E [libj9thr29.so+0xa38e])
(0x00007F17ED0D46BA [libpthread.so.0+0x76ba])
clone+0x6d (0x00007F17EFDD382D [libc.so.6+0x10682d])
---------------------------------------
JVMDUMP039I Processing dump event "gpf", detail "" at 2018/04/11 14:49:42 - 
please wait.
JVMDUMP032I JVM requested System dump using 
'//core.20180411.144942.84.0001.dmp' in response to an event
JVMDUMP030W Cannot write dump to file //core.20180411.144942.84.0001.dmp: 
Permission denied
#0: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x302a94) 
[0x7f17e4f10a94]
#1: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x306b2d) 
[0x7f17e4f14b2d]
#2: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xc82da) 
[0x7f17e4cd62da]
#3: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x22056) 
[0x7f17e6531056]
#4: /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390) [0x7f17ed0de390]
#5: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2c4e1f) 
[0x7f17e4ed2e1f]
#6: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x158c04) 
[0x7f17e4d66c04]
#7: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542d24) 
[0x7f17e5150d24]
#8: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542e0b) 
[0x7f17e5150e0b]
#9: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x54981a) 
[0x7f17e515781a]
#10: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x5494c8) 
[0x7f17e51574c8]
#11: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x548dd2) 
[0x7f17e5156dd2]
#12: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2d7019) 
[0x7f17e4ee5019]
#13: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd31ee) 
[0x7f17e4ce11ee]
#14: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd3e51) 
[0x7f17e4ce1e51]
#15: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#16: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd5ce5) 
[0x7f17e4ce3ce5]
#17: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd668b) 
[0x7f17e4ce468b]
#18: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6d20) 
[0x7f17e4ce4d20]
#19: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6fea) 
[0x7f17e4ce4fea]
#20: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd70b4) 
[0x7f17e4ce50b4]
#21: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#22: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd743f) 
[0x7f17e4ce543f]
#23: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9thr29.so(+0xa38e) 
[0x7f17e699738e]
#24: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f17ed0d46ba]
#25: function clone+0x6d [0x7f17efdd382d]
Unhandled exception
Type=Segmentation error vmState=0x0005ff01
J9Generic_Signal_Number=00000004 Signal_Number=0000000b Error_Value=00000000 
Signal_Code=00000001
Handler1=00007F17E6C26460 Handler2=00007F17E6530E40 
InaccessibleAddress=000000000000001A
RDI=00007F17B1BF70B0 RSI=00007F17C8D50100 RAX=0000000000000012 
RBX=00007F17B1BF7120
RCX=0000000000000000 RDX=00007F17E577BC90 R8=00007F17B1C60670 
R9=00007F17C8D50300
R10=0000000000000011 R11=00007F17E5459071 R12=00007F17C8D526D8 
R13=00007F17B1BF6420
R14=00007F17B1BF6290 R15=00007F17BC0270A0
RIP=00007F17E4ED2E1F GS=0000 FS=0000 RSP=00007F17C8D50150
EFlags=0000000000010216 CS=E033 RBP=00007F17B1BF0000 ERR=0000000000000004
TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=000000000000001A
xmm0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm1 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm4 00000000000000ff (f: 255.000000, d: 1.259867e-321)
xmm5 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
Module=/opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so
Module_base_address=00007F17E4C0E000
Method_being_compiled=org/apache/cassandra/utils/concurrent/Locks.monitorEnterUnsafe(Ljava/lang/Object;)V
Target=2_90_20171215_373586 (Linux 4.4.0-116-generic)
CPU=amd64 (8 logical CPUs) (0xbc2d25000 RAM)
----------- Stack Backtrace -----------
(0x00007F17E4ED2E1F [libj9jit29.so+0x2c4e1f])
(0x00007F17E4D66C04 [libj9jit29.so+0x158c04])
(0x00007F17E5150D24 [libj9jit29.so+0x542d24])
(0x00007F17E5150E0B [libj9jit29.so+0x542e0b])
(0x00007F17E515781A [libj9jit29.so+0x54981a])
(0x00007F17E51574C8 [libj9jit29.so+0x5494c8])
(0x00007F17E5156DD2 [libj9jit29.so+0x548dd2])
(0x00007F17E4EE5019 [libj9jit29.so+0x2d7019])
(0x00007F17E4CE11EE [libj9jit29.so+0xd31ee])
(0x00007F17E4CE1E51 [libj9jit29.so+0xd3e51])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE3CE5 [libj9jit29.so+0xd5ce5])
(0x00007F17E4CE468B [libj9jit29.so+0xd668b])
(0x00007F17E4CE4D20 [libj9jit29.so+0xd6d20])
(0x00007F17E4CE4FEA [libj9jit29.so+0xd6fea])
(0x00007F17E4CE50B4 [libj9jit29.so+0xd70b4])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE543F [libj9jit29.so+0xd743f])
(0x00007F17E699738E [libj9thr29.so+0xa38e])
(0x00007F17ED0D46BA [libpthread.so.0+0x76ba])
clone+0x6d (0x00007F17EFDD382D [libc.so.6+0x10682d])
---------------------------------------
JVMDUMP039I Processing dump event "gpf", detail "" at 2018/04/11 14:49:44 - 
please wait.
JVMPORT030W /proc/sys/kernel/core_pattern setting "|/usr/share/apport/apport %p 
%s %c %d %P" specifies that the core dump is to be piped to an external 
program.  Attempting to rename either core or core.1770.
JVMDUMP012E Error in System dump: The core file created by child process with 
pid = 1770 was not found. Expected to find core file with name "/tmp/core"
JVMDUMP030W Cannot write dump to file //javacore.20180411.144942.84.0003.txt: 
Permission denied
JVMDUMP032I JVM requested Java dump using 
'/tmp/javacore.20180411.144942.84.0003.txt' in response to an event
JVMDUMP032I JVM requested System dump using 
'//core.20180411.144944.84.0002.dmp' in response to an event
JVMDUMP030W Cannot write dump to file //core.20180411.144944.84.0002.dmp: 
Permission denied
JVMDUMP010I Java dump written to /tmp/javacore.20180411.144942.84.0003.txt
ERROR [MemtableFlushWriter:1] 2018-04-11 14:49:51,747 LogTransaction.java:257 - 
Transaction log [mc_txn_flush_95f3d0a0-3d97-11e8-9d33-2f3add552a16.log in 
/var/db/data/data/apim/org_by_owner_url-e1f6b6b03ccb11e887040d4467579f5c] 
indicates txn was not completed, trying to abort it now
ERROR [MemtablePostFlush:1] 2018-04-11 14:49:51,816 CassandraDaemon.java:228 - 
Exception in thread Thread[MemtablePostFlush:1,5,main]
java.lang.RuntimeException: java.io.IOException: Bad file descriptor
        at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:588) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.io.util.FileUtils.appendAndSync(FileUtils.java:567) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplica.append(LogReplica.java:85) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet.lambda$null$5(LogReplicaSet.java:210)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet$$Lambda$155.00000000BC4477D0.perform(Unknown
 Source) ~[na:na]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet.append(LogReplicaSet.java:210) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogFile.addRecord(LogFile.java:303) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.db.lifecycle.LogFile.commit(LogFile.java:250) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction$$Lambda$169.00000000BC44FA10.perform(Unknown
 Source) ~[na:na]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:98) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction.doCommit(LogTransaction.java:389)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LifecycleTransaction.doCommit(LifecycleTransaction.java:218)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1176)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) 
~[na:1.8.0]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) 
[na:1.8.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
 [apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000E0F32EF0.run(Unknown
 Source) ~[na:na]
        at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
        Suppressed: java.lang.IllegalStateException: Attempted to abort a 
committed operation
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:133)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.db.lifecycle.LifecycleTransaction.doAbort(LifecycleTransaction.java:243)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:144)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:193)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.close(Transactional.java:158)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1193)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                ... 6 common frames omitted
Caused by: java.io.IOException: Bad file descriptor
        at sun.nio.ch.FileDispatcherImpl.close0(Native Method) ~[na:1.8.0]
        at sun.nio.ch.FileDispatcherImpl.close(FileDispatcherImpl.java:127) 
~[na:1.8.0]
        at 
sun.nio.ch.FileChannelImpl.implCloseChannel(FileChannelImpl.java:153) 
~[na:1.8.0]
        at 
java.nio.channels.spi.AbstractInterruptibleChannel.close(AbstractInterruptibleChannel.java:126)
 ~[na:1.8.0]
        at java.nio.channels.Channels$1.close(Channels.java:189) ~[na:1.8.0]
        at sun.nio.cs.StreamEncoder.implClose(StreamEncoder.java:332) 
~[na:1.8.0]
        at sun.nio.cs.StreamEncoder.close(StreamEncoder.java:161) ~[na:1.8.0]
        at java.io.OutputStreamWriter.close(OutputStreamWriter.java:295) 
~[na:1.8.0]
        at java.io.BufferedWriter.close(BufferedWriter.java:277) ~[na:1.8.0]
        at java.nio.file.Files.write(Files.java:3368) ~[na:1.8.0]
        at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:581) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        ... 24 common frames omitted
JVMPORT030W /proc/sys/kernel/core_pattern setting "|/usr/share/apport/apport %p 
%s %c %d %P" specifies that the core dump is to be piped to an external 
program.  Attempting to rename either core or core.1781.
JVMDUMP012E Error in System dump: The core file created by child process with 
pid = 1781 was not found. Expected to find core file with name "/tmp/core"
JVMDUMP030W Cannot write dump to file //javacore.20180411.144944.84.0005.txt: 
Permission denied
JVMDUMP032I JVM requested Java dump using 
'/tmp/javacore.20180411.144944.84.0005.txt' in response to an event
JVMDUMP032I JVM requested Snap dump using '//Snap.20180411.144942.84.0004.trc' 
in response to an event
JVMDUMP030W Cannot write dump to file //Snap.20180411.144942.84.0004.trc: 
Permission denied
JVMDUMP010I Snap dump written to /tmp/Snap.20180411.144942.84.0004.trc
JVMDUMP030W Cannot write dump to file //jitdump.20180411.144942.84.0006.dmp: 
Permission denied
JVMDUMP007I 
#JITDUMP:  vmThread=0000000001B3B800 Recursive crash occurred. Aborting JIT 
dump.JVMDUMP010I JIT dump written to /tmp/jitdump.20180411.144942.84.0006.dmp
WARN  [MemtableFlushWriter:1] 2018-04-11 14:49:58,241 NativeLibrary.java:328 - 
fsync(3084) failed, errorno (22) {}
com.sun.jna.LastErrorException: [22] Invalid argument
        at org.apache.cassandra.utils.NativeLibraryLinux.fsync(Native Method) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.NativeLibraryLinux.callFsync(NativeLibraryLinux.java:107)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.NativeLibrary.trySync(NativeLibrary.java:317) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplica.syncDirectory(LogReplica.java:96) 
[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplica.delete(LogReplica.java:102) 
[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet$$Lambda$175.00000000BC451FC0.perform(Unknown
 Source) ~[na:na]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet.delete(LogReplicaSet.java:92) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogFile.removeUnfinishedLeftovers(LogFile.java:128)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.run(LogTransaction.java:263)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.tidy(LogTransaction.java:242)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Ref$GlobalState.release(Ref.java:321) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Ref$State.ensureReleased(Ref.java:199) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Ref.ensureReleased(Ref.java:119) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction.complete(LogTransaction.java:377)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction.doCommit(LogTransaction.java:389)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LifecycleTransaction.doCommit(LifecycleTransaction.java:218)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1176)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) 
~[na:1.8.0]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) 
~[na:1.8.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000E0F32EF0.run(Unknown
 Source) ~[na:na]
        at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
JVMDUMP013I Processed dump event "gpf", detail "".

 

  was:
Running Cassandra with a Rook Ceph filesystem within Kubernetes.  During the 
startup, the following Warnings in the debug log pop up and then Cassandra 
crashes shortly after and restarts:

WARN [MemtableFlushWriter:2] 2018-04-11 14:34:42,748 NativeLibrary.java:328 - 
fsync(666) failed, errorno (22) {}
com.sun.jna.LastErrorException: [22] Invalid argument
 at org.apache.cassandra.utils.NativeLibraryLinux.fsync(Native Method) 
~[apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.utils.NativeLibraryLinux.callFsync(NativeLibraryLinux.java:107)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
 at org.apache.cassandra.utils.NativeLibrary.trySync(NativeLibrary.java:317) 
~[apache-cassandra-3.11.0.jar:3.11.0]
 at org.apache.cassandra.utils.SyncUtil.trySync(SyncUtil.java:179) 
[apache-cassandra-3.11.0.jar:3.11.0]
 at org.apache.cassandra.utils.SyncUtil.trySyncDir(SyncUtil.java:190) 
[apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.util.SequentialWriter.openChannel(SequentialWriter.java:107)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.util.SequentialWriter.<init>(SequentialWriter.java:141) 
[apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.big.BigTableWriter.writeMetadata(BigTableWriter.java:402)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.big.BigTableWriter.access$300(BigTableWriter.java:53)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.big.BigTableWriter$TransactionalProxy.doPrepare(BigTableWriter.java:368)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.prepareToCommit(Transactional.java:173)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.format.SSTableWriter.prepareToCommit(SSTableWriter.java:281)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter.prepareToCommit(SimpleSSTableMultiWriter.java:101)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1153)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) 
[na:1.8.0]
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) 
[na:1.8.0]
 at 
org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
 [apache-cassandra-3.11.0.jar:3.11.0]
 at 
org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000BCF32600.run(Unknown
 Source) ~[na:na]
 at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]

 
Syslog shows the following 
(logs-from-cassandra-in-r97bb66e967-apiconnect-cc-0.txt):

INFO  [main] 2018-04-11 14:49:01,848 ColumnFamilyStore.java:406 - Initializing 
apim.ur_to_op_by_op
INFO  [MemoryMXBean notification dispatcher] 2018-04-11 14:49:25,889 
GCInspector.java:284 - global GC in 206ms.  class storage: 28700680 -> 
28692744; miscellaneous non-heap storage: 49871216 -> 53570176; 
nursery-allocate: 1296878920 -> 149116672; tenured-SOA: 140321968 -> 139143760
#0: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x302a94) 
[0x7f17e4f10a94]
#1: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x306b2d) 
[0x7f17e4f14b2d]
#2: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xc82da) 
[0x7f17e4cd62da]
#3: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x22056) 
[0x7f17e6531056]
#4: /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390) [0x7f17ed0de390]
#5: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2c4e1f) 
[0x7f17e4ed2e1f]
#6: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x158c04) 
[0x7f17e4d66c04]
#7: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542d24) 
[0x7f17e5150d24]
#8: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542e0b) 
[0x7f17e5150e0b]
#9: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x54981a) 
[0x7f17e515781a]
#10: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x5494c8) 
[0x7f17e51574c8]
#11: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x548dd2) 
[0x7f17e5156dd2]
#12: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2d7019) 
[0x7f17e4ee5019]
#13: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd31ee) 
[0x7f17e4ce11ee]
#14: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd3e51) 
[0x7f17e4ce1e51]
#15: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#16: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd5ce5) 
[0x7f17e4ce3ce5]
#17: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd668b) 
[0x7f17e4ce468b]
#18: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6d20) 
[0x7f17e4ce4d20]
#19: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6fea) 
[0x7f17e4ce4fea]
#20: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd70b4) 
[0x7f17e4ce50b4]
#21: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#22: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd743f) 
[0x7f17e4ce543f]
#23: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9thr29.so(+0xa38e) 
[0x7f17e699738e]
#24: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f17ed0d46ba]
#25: function clone+0x6d [0x7f17efdd382d]
Unhandled exception
Type=Segmentation error vmState=0x0005ff01
J9Generic_Signal_Number=00000004 Signal_Number=0000000b Error_Value=00000000 
Signal_Code=00000001
Handler1=00007F17E6C26460 Handler2=00007F17E6530E40 
InaccessibleAddress=000000000000001A
RDI=00007F17B50070B0 RSI=00007F17C8E51100 RAX=0000000000000012 
RBX=00007F17B5007120
RCX=0000000000000000 RDX=00007F17E577BC90 R8=00007F17B5070670 
R9=0000000000000000
R10=0000000000000011 R11=00007F17E5459071 R12=00007F17C8E536D8 
R13=00007F17B5006420
R14=00007F17B5006290 R15=00007F17C4016CB0
RIP=00007F17E4ED2E1F GS=0000 FS=0000 RSP=00007F17C8E51150
EFlags=0000000000010216 CS=E033 RBP=00007F17B5000000 ERR=0000000000000004
TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=000000000000001A
xmm0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm1 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm4 00000000000000ff (f: 255.000000, d: 1.259867e-321)
xmm5 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
Module=/opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so
Module_base_address=00007F17E4C0E000
Method_being_compiled=org/apache/cassandra/utils/concurrent/Locks.monitorExitUnsafe(Ljava/lang/Object;)V
Target=2_90_20171215_373586 (Linux 4.4.0-116-generic)
CPU=amd64 (8 logical CPUs) (0xbc2d25000 RAM)
----------- Stack Backtrace -----------
(0x00007F17E4ED2E1F [libj9jit29.so+0x2c4e1f])
(0x00007F17E4D66C04 [libj9jit29.so+0x158c04])
(0x00007F17E5150D24 [libj9jit29.so+0x542d24])
(0x00007F17E5150E0B [libj9jit29.so+0x542e0b])
(0x00007F17E515781A [libj9jit29.so+0x54981a])
(0x00007F17E51574C8 [libj9jit29.so+0x5494c8])
(0x00007F17E5156DD2 [libj9jit29.so+0x548dd2])
(0x00007F17E4EE5019 [libj9jit29.so+0x2d7019])
(0x00007F17E4CE11EE [libj9jit29.so+0xd31ee])
(0x00007F17E4CE1E51 [libj9jit29.so+0xd3e51])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE3CE5 [libj9jit29.so+0xd5ce5])
(0x00007F17E4CE468B [libj9jit29.so+0xd668b])
(0x00007F17E4CE4D20 [libj9jit29.so+0xd6d20])
(0x00007F17E4CE4FEA [libj9jit29.so+0xd6fea])
(0x00007F17E4CE50B4 [libj9jit29.so+0xd70b4])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE543F [libj9jit29.so+0xd743f])
(0x00007F17E699738E [libj9thr29.so+0xa38e])
(0x00007F17ED0D46BA [libpthread.so.0+0x76ba])
clone+0x6d (0x00007F17EFDD382D [libc.so.6+0x10682d])
---------------------------------------
JVMDUMP039I Processing dump event "gpf", detail "" at 2018/04/11 14:49:42 - 
please wait.
JVMDUMP032I JVM requested System dump using 
'//core.20180411.144942.84.0001.dmp' in response to an event
JVMDUMP030W Cannot write dump to file //core.20180411.144942.84.0001.dmp: 
Permission denied
#0: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x302a94) 
[0x7f17e4f10a94]
#1: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x306b2d) 
[0x7f17e4f14b2d]
#2: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xc82da) 
[0x7f17e4cd62da]
#3: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x22056) 
[0x7f17e6531056]
#4: /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390) [0x7f17ed0de390]
#5: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2c4e1f) 
[0x7f17e4ed2e1f]
#6: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x158c04) 
[0x7f17e4d66c04]
#7: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542d24) 
[0x7f17e5150d24]
#8: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542e0b) 
[0x7f17e5150e0b]
#9: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x54981a) 
[0x7f17e515781a]
#10: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x5494c8) 
[0x7f17e51574c8]
#11: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x548dd2) 
[0x7f17e5156dd2]
#12: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2d7019) 
[0x7f17e4ee5019]
#13: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd31ee) 
[0x7f17e4ce11ee]
#14: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd3e51) 
[0x7f17e4ce1e51]
#15: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#16: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd5ce5) 
[0x7f17e4ce3ce5]
#17: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd668b) 
[0x7f17e4ce468b]
#18: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6d20) 
[0x7f17e4ce4d20]
#19: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6fea) 
[0x7f17e4ce4fea]
#20: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd70b4) 
[0x7f17e4ce50b4]
#21: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
[0x7f17e65313e5]
#22: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd743f) 
[0x7f17e4ce543f]
#23: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9thr29.so(+0xa38e) 
[0x7f17e699738e]
#24: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f17ed0d46ba]
#25: function clone+0x6d [0x7f17efdd382d]
Unhandled exception
Type=Segmentation error vmState=0x0005ff01
J9Generic_Signal_Number=00000004 Signal_Number=0000000b Error_Value=00000000 
Signal_Code=00000001
Handler1=00007F17E6C26460 Handler2=00007F17E6530E40 
InaccessibleAddress=000000000000001A
RDI=00007F17B1BF70B0 RSI=00007F17C8D50100 RAX=0000000000000012 
RBX=00007F17B1BF7120
RCX=0000000000000000 RDX=00007F17E577BC90 R8=00007F17B1C60670 
R9=00007F17C8D50300
R10=0000000000000011 R11=00007F17E5459071 R12=00007F17C8D526D8 
R13=00007F17B1BF6420
R14=00007F17B1BF6290 R15=00007F17BC0270A0
RIP=00007F17E4ED2E1F GS=0000 FS=0000 RSP=00007F17C8D50150
EFlags=0000000000010216 CS=E033 RBP=00007F17B1BF0000 ERR=0000000000000004
TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=000000000000001A
xmm0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm1 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm4 00000000000000ff (f: 255.000000, d: 1.259867e-321)
xmm5 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
Module=/opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so
Module_base_address=00007F17E4C0E000
Method_being_compiled=org/apache/cassandra/utils/concurrent/Locks.monitorEnterUnsafe(Ljava/lang/Object;)V
Target=2_90_20171215_373586 (Linux 4.4.0-116-generic)
CPU=amd64 (8 logical CPUs) (0xbc2d25000 RAM)
----------- Stack Backtrace -----------
(0x00007F17E4ED2E1F [libj9jit29.so+0x2c4e1f])
(0x00007F17E4D66C04 [libj9jit29.so+0x158c04])
(0x00007F17E5150D24 [libj9jit29.so+0x542d24])
(0x00007F17E5150E0B [libj9jit29.so+0x542e0b])
(0x00007F17E515781A [libj9jit29.so+0x54981a])
(0x00007F17E51574C8 [libj9jit29.so+0x5494c8])
(0x00007F17E5156DD2 [libj9jit29.so+0x548dd2])
(0x00007F17E4EE5019 [libj9jit29.so+0x2d7019])
(0x00007F17E4CE11EE [libj9jit29.so+0xd31ee])
(0x00007F17E4CE1E51 [libj9jit29.so+0xd3e51])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE3CE5 [libj9jit29.so+0xd5ce5])
(0x00007F17E4CE468B [libj9jit29.so+0xd668b])
(0x00007F17E4CE4D20 [libj9jit29.so+0xd6d20])
(0x00007F17E4CE4FEA [libj9jit29.so+0xd6fea])
(0x00007F17E4CE50B4 [libj9jit29.so+0xd70b4])
(0x00007F17E65313E5 [libj9prt29.so+0x223e5])
(0x00007F17E4CE543F [libj9jit29.so+0xd743f])
(0x00007F17E699738E [libj9thr29.so+0xa38e])
(0x00007F17ED0D46BA [libpthread.so.0+0x76ba])
clone+0x6d (0x00007F17EFDD382D [libc.so.6+0x10682d])
---------------------------------------
JVMDUMP039I Processing dump event "gpf", detail "" at 2018/04/11 14:49:44 - 
please wait.
JVMPORT030W /proc/sys/kernel/core_pattern setting "|/usr/share/apport/apport %p 
%s %c %d %P" specifies that the core dump is to be piped to an external 
program.  Attempting to rename either core or core.1770.
JVMDUMP012E Error in System dump: The core file created by child process with 
pid = 1770 was not found. Expected to find core file with name "/tmp/core"
JVMDUMP030W Cannot write dump to file //javacore.20180411.144942.84.0003.txt: 
Permission denied
JVMDUMP032I JVM requested Java dump using 
'/tmp/javacore.20180411.144942.84.0003.txt' in response to an event
JVMDUMP032I JVM requested System dump using 
'//core.20180411.144944.84.0002.dmp' in response to an event
JVMDUMP030W Cannot write dump to file //core.20180411.144944.84.0002.dmp: 
Permission denied
JVMDUMP010I Java dump written to /tmp/javacore.20180411.144942.84.0003.txt
ERROR [MemtableFlushWriter:1] 2018-04-11 14:49:51,747 LogTransaction.java:257 - 
Transaction log [mc_txn_flush_95f3d0a0-3d97-11e8-9d33-2f3add552a16.log in 
/var/db/data/data/apim/org_by_owner_url-e1f6b6b03ccb11e887040d4467579f5c] 
indicates txn was not completed, trying to abort it now
ERROR [MemtablePostFlush:1] 2018-04-11 14:49:51,816 CassandraDaemon.java:228 - 
Exception in thread Thread[MemtablePostFlush:1,5,main]
java.lang.RuntimeException: java.io.IOException: Bad file descriptor
        at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:588) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.io.util.FileUtils.appendAndSync(FileUtils.java:567) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplica.append(LogReplica.java:85) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet.lambda$null$5(LogReplicaSet.java:210)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet$$Lambda$155.00000000BC4477D0.perform(Unknown
 Source) ~[na:na]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet.append(LogReplicaSet.java:210) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogFile.addRecord(LogFile.java:303) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.db.lifecycle.LogFile.commit(LogFile.java:250) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction$$Lambda$169.00000000BC44FA10.perform(Unknown
 Source) ~[na:na]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:98) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction.doCommit(LogTransaction.java:389)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LifecycleTransaction.doCommit(LifecycleTransaction.java:218)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1176)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) 
~[na:1.8.0]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) 
[na:1.8.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
 [apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000E0F32EF0.run(Unknown
 Source) ~[na:na]
        at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
        Suppressed: java.lang.IllegalStateException: Attempted to abort a 
committed operation
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:133)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.db.lifecycle.LifecycleTransaction.doAbort(LifecycleTransaction.java:243)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:144)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:193)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.close(Transactional.java:158)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1193)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
                ... 6 common frames omitted
Caused by: java.io.IOException: Bad file descriptor
        at sun.nio.ch.FileDispatcherImpl.close0(Native Method) ~[na:1.8.0]
        at sun.nio.ch.FileDispatcherImpl.close(FileDispatcherImpl.java:127) 
~[na:1.8.0]
        at 
sun.nio.ch.FileChannelImpl.implCloseChannel(FileChannelImpl.java:153) 
~[na:1.8.0]
        at 
java.nio.channels.spi.AbstractInterruptibleChannel.close(AbstractInterruptibleChannel.java:126)
 ~[na:1.8.0]
        at java.nio.channels.Channels$1.close(Channels.java:189) ~[na:1.8.0]
        at sun.nio.cs.StreamEncoder.implClose(StreamEncoder.java:332) 
~[na:1.8.0]
        at sun.nio.cs.StreamEncoder.close(StreamEncoder.java:161) ~[na:1.8.0]
        at java.io.OutputStreamWriter.close(OutputStreamWriter.java:295) 
~[na:1.8.0]
        at java.io.BufferedWriter.close(BufferedWriter.java:277) ~[na:1.8.0]
        at java.nio.file.Files.write(Files.java:3368) ~[na:1.8.0]
        at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:581) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        ... 24 common frames omitted
JVMPORT030W /proc/sys/kernel/core_pattern setting "|/usr/share/apport/apport %p 
%s %c %d %P" specifies that the core dump is to be piped to an external 
program.  Attempting to rename either core or core.1781.
JVMDUMP012E Error in System dump: The core file created by child process with 
pid = 1781 was not found. Expected to find core file with name "/tmp/core"
JVMDUMP030W Cannot write dump to file //javacore.20180411.144944.84.0005.txt: 
Permission denied
JVMDUMP032I JVM requested Java dump using 
'/tmp/javacore.20180411.144944.84.0005.txt' in response to an event
JVMDUMP032I JVM requested Snap dump using '//Snap.20180411.144942.84.0004.trc' 
in response to an event
JVMDUMP030W Cannot write dump to file //Snap.20180411.144942.84.0004.trc: 
Permission denied
JVMDUMP010I Snap dump written to /tmp/Snap.20180411.144942.84.0004.trc
JVMDUMP030W Cannot write dump to file //jitdump.20180411.144942.84.0006.dmp: 
Permission denied
JVMDUMP007I 
#JITDUMP:  vmThread=0000000001B3B800 Recursive crash occurred. Aborting JIT 
dump.JVMDUMP010I JIT dump written to /tmp/jitdump.20180411.144942.84.0006.dmp
WARN  [MemtableFlushWriter:1] 2018-04-11 14:49:58,241 NativeLibrary.java:328 - 
fsync(3084) failed, errorno (22) {}
com.sun.jna.LastErrorException: [22] Invalid argument
        at org.apache.cassandra.utils.NativeLibraryLinux.fsync(Native Method) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.NativeLibraryLinux.callFsync(NativeLibraryLinux.java:107)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.NativeLibrary.trySync(NativeLibrary.java:317) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplica.syncDirectory(LogReplica.java:96) 
[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplica.delete(LogReplica.java:102) 
[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet$$Lambda$175.00000000BC451FC0.perform(Unknown
 Source) ~[na:na]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogReplicaSet.delete(LogReplicaSet.java:92) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogFile.removeUnfinishedLeftovers(LogFile.java:128)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.run(LogTransaction.java:263)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.tidy(LogTransaction.java:242)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Ref$GlobalState.release(Ref.java:321) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Ref$State.ensureReleased(Ref.java:199) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Ref.ensureReleased(Ref.java:119) 
~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction.complete(LogTransaction.java:377)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LogTransaction.doCommit(LogTransaction.java:389)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.lifecycle.LifecycleTransaction.doCommit(LifecycleTransaction.java:218)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1176)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) 
~[na:1.8.0]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) 
~[na:1.8.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
 ~[apache-cassandra-3.11.0.jar:3.11.0]
        at 
org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000E0F32EF0.run(Unknown
 Source) ~[na:na]
        at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
JVMDUMP013I Processed dump event "gpf", detail "".

 


> Cassandra crashes after fsync exception
> ---------------------------------------
>
>                 Key: CASSANDRA-14380
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-14380
>             Project: Cassandra
>          Issue Type: Bug
>            Reporter: Adam Geiger
>            Priority: Critical
>         Attachments: debug.log, debug.log.1.zip, 
> logs-from-cassandra-in-r97bb66e967-apiconnect-cc-0.txt
>
>
> Running Cassandra with a Rook Ceph filesystem within Kubernetes.  During the 
> startup, the following Warnings in the debug log pop up and then Cassandra 
> crashes shortly after and restarts.  It looks like before hitting this error, 
> it is doing a lot of writing and flushing
> WARN [MemtableFlushWriter:2] 2018-04-11 14:34:42,748 NativeLibrary.java:328 - 
> fsync(666) failed, errorno (22) {}
> com.sun.jna.LastErrorException: [22] Invalid argument
>  at org.apache.cassandra.utils.NativeLibraryLinux.fsync(Native Method) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.utils.NativeLibraryLinux.callFsync(NativeLibraryLinux.java:107)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>  at org.apache.cassandra.utils.NativeLibrary.trySync(NativeLibrary.java:317) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>  at org.apache.cassandra.utils.SyncUtil.trySync(SyncUtil.java:179) 
> [apache-cassandra-3.11.0.jar:3.11.0]
>  at org.apache.cassandra.utils.SyncUtil.trySyncDir(SyncUtil.java:190) 
> [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.util.SequentialWriter.openChannel(SequentialWriter.java:107)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.util.SequentialWriter.<init>(SequentialWriter.java:141)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.sstable.format.big.BigTableWriter.writeMetadata(BigTableWriter.java:402)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.sstable.format.big.BigTableWriter.access$300(BigTableWriter.java:53)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.sstable.format.big.BigTableWriter$TransactionalProxy.doPrepare(BigTableWriter.java:368)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.prepareToCommit(Transactional.java:173)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.sstable.format.SSTableWriter.prepareToCommit(SSTableWriter.java:281)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter.prepareToCommit(SimpleSSTableMultiWriter.java:101)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1153)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160)
>  [na:1.8.0]
>  at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
>  [na:1.8.0]
>  at 
> org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>  at 
> org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000BCF32600.run(Unknown
>  Source) ~[na:na]
>  at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
>  
> Syslog shows the following 
> (logs-from-cassandra-in-r97bb66e967-apiconnect-cc-0.txt):
> INFO  [main] 2018-04-11 14:49:01,848 ColumnFamilyStore.java:406 - 
> Initializing apim.ur_to_op_by_op
> INFO  [MemoryMXBean notification dispatcher] 2018-04-11 14:49:25,889 
> GCInspector.java:284 - global GC in 206ms.  class storage: 28700680 -> 
> 28692744; miscellaneous non-heap storage: 49871216 -> 53570176; 
> nursery-allocate: 1296878920 -> 149116672; tenured-SOA: 140321968 -> 139143760
> #0: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x302a94) 
> [0x7f17e4f10a94]
> #1: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x306b2d) 
> [0x7f17e4f14b2d]
> #2: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xc82da) 
> [0x7f17e4cd62da]
> #3: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x22056) 
> [0x7f17e6531056]
> #4: /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390) [0x7f17ed0de390]
> #5: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2c4e1f) 
> [0x7f17e4ed2e1f]
> #6: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x158c04) 
> [0x7f17e4d66c04]
> #7: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542d24) 
> [0x7f17e5150d24]
> #8: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542e0b) 
> [0x7f17e5150e0b]
> #9: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x54981a) 
> [0x7f17e515781a]
> #10: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x5494c8) 
> [0x7f17e51574c8]
> #11: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x548dd2) 
> [0x7f17e5156dd2]
> #12: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2d7019) 
> [0x7f17e4ee5019]
> #13: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd31ee) 
> [0x7f17e4ce11ee]
> #14: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd3e51) 
> [0x7f17e4ce1e51]
> #15: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
> [0x7f17e65313e5]
> #16: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd5ce5) 
> [0x7f17e4ce3ce5]
> #17: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd668b) 
> [0x7f17e4ce468b]
> #18: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6d20) 
> [0x7f17e4ce4d20]
> #19: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6fea) 
> [0x7f17e4ce4fea]
> #20: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd70b4) 
> [0x7f17e4ce50b4]
> #21: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
> [0x7f17e65313e5]
> #22: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd743f) 
> [0x7f17e4ce543f]
> #23: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9thr29.so(+0xa38e) 
> [0x7f17e699738e]
> #24: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f17ed0d46ba]
> #25: function clone+0x6d [0x7f17efdd382d]
> Unhandled exception
> Type=Segmentation error vmState=0x0005ff01
> J9Generic_Signal_Number=00000004 Signal_Number=0000000b Error_Value=00000000 
> Signal_Code=00000001
> Handler1=00007F17E6C26460 Handler2=00007F17E6530E40 
> InaccessibleAddress=000000000000001A
> RDI=00007F17B50070B0 RSI=00007F17C8E51100 RAX=0000000000000012 
> RBX=00007F17B5007120
> RCX=0000000000000000 RDX=00007F17E577BC90 R8=00007F17B5070670 
> R9=0000000000000000
> R10=0000000000000011 R11=00007F17E5459071 R12=00007F17C8E536D8 
> R13=00007F17B5006420
> R14=00007F17B5006290 R15=00007F17C4016CB0
> RIP=00007F17E4ED2E1F GS=0000 FS=0000 RSP=00007F17C8E51150
> EFlags=0000000000010216 CS=E033 RBP=00007F17B5000000 ERR=0000000000000004
> TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=000000000000001A
> xmm0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm1 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm4 00000000000000ff (f: 255.000000, d: 1.259867e-321)
> xmm5 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> Module=/opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so
> Module_base_address=00007F17E4C0E000
> Method_being_compiled=org/apache/cassandra/utils/concurrent/Locks.monitorExitUnsafe(Ljava/lang/Object;)V
> Target=2_90_20171215_373586 (Linux 4.4.0-116-generic)
> CPU=amd64 (8 logical CPUs) (0xbc2d25000 RAM)
> ----------- Stack Backtrace -----------
> (0x00007F17E4ED2E1F [libj9jit29.so+0x2c4e1f])
> (0x00007F17E4D66C04 [libj9jit29.so+0x158c04])
> (0x00007F17E5150D24 [libj9jit29.so+0x542d24])
> (0x00007F17E5150E0B [libj9jit29.so+0x542e0b])
> (0x00007F17E515781A [libj9jit29.so+0x54981a])
> (0x00007F17E51574C8 [libj9jit29.so+0x5494c8])
> (0x00007F17E5156DD2 [libj9jit29.so+0x548dd2])
> (0x00007F17E4EE5019 [libj9jit29.so+0x2d7019])
> (0x00007F17E4CE11EE [libj9jit29.so+0xd31ee])
> (0x00007F17E4CE1E51 [libj9jit29.so+0xd3e51])
> (0x00007F17E65313E5 [libj9prt29.so+0x223e5])
> (0x00007F17E4CE3CE5 [libj9jit29.so+0xd5ce5])
> (0x00007F17E4CE468B [libj9jit29.so+0xd668b])
> (0x00007F17E4CE4D20 [libj9jit29.so+0xd6d20])
> (0x00007F17E4CE4FEA [libj9jit29.so+0xd6fea])
> (0x00007F17E4CE50B4 [libj9jit29.so+0xd70b4])
> (0x00007F17E65313E5 [libj9prt29.so+0x223e5])
> (0x00007F17E4CE543F [libj9jit29.so+0xd743f])
> (0x00007F17E699738E [libj9thr29.so+0xa38e])
> (0x00007F17ED0D46BA [libpthread.so.0+0x76ba])
> clone+0x6d (0x00007F17EFDD382D [libc.so.6+0x10682d])
> ---------------------------------------
> JVMDUMP039I Processing dump event "gpf", detail "" at 2018/04/11 14:49:42 - 
> please wait.
> JVMDUMP032I JVM requested System dump using 
> '//core.20180411.144942.84.0001.dmp' in response to an event
> JVMDUMP030W Cannot write dump to file //core.20180411.144942.84.0001.dmp: 
> Permission denied
> #0: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x302a94) 
> [0x7f17e4f10a94]
> #1: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x306b2d) 
> [0x7f17e4f14b2d]
> #2: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xc82da) 
> [0x7f17e4cd62da]
> #3: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x22056) 
> [0x7f17e6531056]
> #4: /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390) [0x7f17ed0de390]
> #5: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2c4e1f) 
> [0x7f17e4ed2e1f]
> #6: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x158c04) 
> [0x7f17e4d66c04]
> #7: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542d24) 
> [0x7f17e5150d24]
> #8: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x542e0b) 
> [0x7f17e5150e0b]
> #9: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x54981a) 
> [0x7f17e515781a]
> #10: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x5494c8) 
> [0x7f17e51574c8]
> #11: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x548dd2) 
> [0x7f17e5156dd2]
> #12: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0x2d7019) 
> [0x7f17e4ee5019]
> #13: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd31ee) 
> [0x7f17e4ce11ee]
> #14: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd3e51) 
> [0x7f17e4ce1e51]
> #15: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
> [0x7f17e65313e5]
> #16: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd5ce5) 
> [0x7f17e4ce3ce5]
> #17: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd668b) 
> [0x7f17e4ce468b]
> #18: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6d20) 
> [0x7f17e4ce4d20]
> #19: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd6fea) 
> [0x7f17e4ce4fea]
> #20: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd70b4) 
> [0x7f17e4ce50b4]
> #21: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9prt29.so(+0x223e5) 
> [0x7f17e65313e5]
> #22: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so(+0xd743f) 
> [0x7f17e4ce543f]
> #23: /opt/ibm/java/jre/lib/amd64/compressedrefs/libj9thr29.so(+0xa38e) 
> [0x7f17e699738e]
> #24: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f17ed0d46ba]
> #25: function clone+0x6d [0x7f17efdd382d]
> Unhandled exception
> Type=Segmentation error vmState=0x0005ff01
> J9Generic_Signal_Number=00000004 Signal_Number=0000000b Error_Value=00000000 
> Signal_Code=00000001
> Handler1=00007F17E6C26460 Handler2=00007F17E6530E40 
> InaccessibleAddress=000000000000001A
> RDI=00007F17B1BF70B0 RSI=00007F17C8D50100 RAX=0000000000000012 
> RBX=00007F17B1BF7120
> RCX=0000000000000000 RDX=00007F17E577BC90 R8=00007F17B1C60670 
> R9=00007F17C8D50300
> R10=0000000000000011 R11=00007F17E5459071 R12=00007F17C8D526D8 
> R13=00007F17B1BF6420
> R14=00007F17B1BF6290 R15=00007F17BC0270A0
> RIP=00007F17E4ED2E1F GS=0000 FS=0000 RSP=00007F17C8D50150
> EFlags=0000000000010216 CS=E033 RBP=00007F17B1BF0000 ERR=0000000000000004
> TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=000000000000001A
> xmm0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm1 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm4 00000000000000ff (f: 255.000000, d: 1.259867e-321)
> xmm5 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
> Module=/opt/ibm/java/jre/lib/amd64/compressedrefs/libj9jit29.so
> Module_base_address=00007F17E4C0E000
> Method_being_compiled=org/apache/cassandra/utils/concurrent/Locks.monitorEnterUnsafe(Ljava/lang/Object;)V
> Target=2_90_20171215_373586 (Linux 4.4.0-116-generic)
> CPU=amd64 (8 logical CPUs) (0xbc2d25000 RAM)
> ----------- Stack Backtrace -----------
> (0x00007F17E4ED2E1F [libj9jit29.so+0x2c4e1f])
> (0x00007F17E4D66C04 [libj9jit29.so+0x158c04])
> (0x00007F17E5150D24 [libj9jit29.so+0x542d24])
> (0x00007F17E5150E0B [libj9jit29.so+0x542e0b])
> (0x00007F17E515781A [libj9jit29.so+0x54981a])
> (0x00007F17E51574C8 [libj9jit29.so+0x5494c8])
> (0x00007F17E5156DD2 [libj9jit29.so+0x548dd2])
> (0x00007F17E4EE5019 [libj9jit29.so+0x2d7019])
> (0x00007F17E4CE11EE [libj9jit29.so+0xd31ee])
> (0x00007F17E4CE1E51 [libj9jit29.so+0xd3e51])
> (0x00007F17E65313E5 [libj9prt29.so+0x223e5])
> (0x00007F17E4CE3CE5 [libj9jit29.so+0xd5ce5])
> (0x00007F17E4CE468B [libj9jit29.so+0xd668b])
> (0x00007F17E4CE4D20 [libj9jit29.so+0xd6d20])
> (0x00007F17E4CE4FEA [libj9jit29.so+0xd6fea])
> (0x00007F17E4CE50B4 [libj9jit29.so+0xd70b4])
> (0x00007F17E65313E5 [libj9prt29.so+0x223e5])
> (0x00007F17E4CE543F [libj9jit29.so+0xd743f])
> (0x00007F17E699738E [libj9thr29.so+0xa38e])
> (0x00007F17ED0D46BA [libpthread.so.0+0x76ba])
> clone+0x6d (0x00007F17EFDD382D [libc.so.6+0x10682d])
> ---------------------------------------
> JVMDUMP039I Processing dump event "gpf", detail "" at 2018/04/11 14:49:44 - 
> please wait.
> JVMPORT030W /proc/sys/kernel/core_pattern setting "|/usr/share/apport/apport 
> %p %s %c %d %P" specifies that the core dump is to be piped to an external 
> program.  Attempting to rename either core or core.1770.
> JVMDUMP012E Error in System dump: The core file created by child process with 
> pid = 1770 was not found. Expected to find core file with name "/tmp/core"
> JVMDUMP030W Cannot write dump to file //javacore.20180411.144942.84.0003.txt: 
> Permission denied
> JVMDUMP032I JVM requested Java dump using 
> '/tmp/javacore.20180411.144942.84.0003.txt' in response to an event
> JVMDUMP032I JVM requested System dump using 
> '//core.20180411.144944.84.0002.dmp' in response to an event
> JVMDUMP030W Cannot write dump to file //core.20180411.144944.84.0002.dmp: 
> Permission denied
> JVMDUMP010I Java dump written to /tmp/javacore.20180411.144942.84.0003.txt
> ERROR [MemtableFlushWriter:1] 2018-04-11 14:49:51,747 LogTransaction.java:257 
> - Transaction log [mc_txn_flush_95f3d0a0-3d97-11e8-9d33-2f3add552a16.log in 
> /var/db/data/data/apim/org_by_owner_url-e1f6b6b03ccb11e887040d4467579f5c] 
> indicates txn was not completed, trying to abort it now
> ERROR [MemtablePostFlush:1] 2018-04-11 14:49:51,816 CassandraDaemon.java:228 
> - Exception in thread Thread[MemtablePostFlush:1,5,main]
> java.lang.RuntimeException: java.io.IOException: Bad file descriptor
>       at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:588) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.io.util.FileUtils.appendAndSync(FileUtils.java:567) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplica.append(LogReplica.java:85) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplicaSet.lambda$null$5(LogReplicaSet.java:210)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplicaSet$$Lambda$155.00000000BC4477D0.perform(Unknown
>  Source) ~[na:na]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplicaSet.append(LogReplicaSet.java:210)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogFile.addRecord(LogFile.java:303) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at org.apache.cassandra.db.lifecycle.LogFile.commit(LogFile.java:250) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogTransaction$$Lambda$169.00000000BC44FA10.perform(Unknown
>  Source) ~[na:na]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:98) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogTransaction.doCommit(LogTransaction.java:389)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LifecycleTransaction.doCommit(LifecycleTransaction.java:218)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1176)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160)
>  ~[na:1.8.0]
>       at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
>  [na:1.8.0]
>       at 
> org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000E0F32EF0.run(Unknown
>  Source) ~[na:na]
>       at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
>       Suppressed: java.lang.IllegalStateException: Attempted to abort a 
> committed operation
>               at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:133)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>               at 
> org.apache.cassandra.db.lifecycle.LifecycleTransaction.doAbort(LifecycleTransaction.java:243)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>               at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:144)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>               at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:193)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>               at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.close(Transactional.java:158)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>               at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1193)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>               ... 6 common frames omitted
> Caused by: java.io.IOException: Bad file descriptor
>       at sun.nio.ch.FileDispatcherImpl.close0(Native Method) ~[na:1.8.0]
>       at sun.nio.ch.FileDispatcherImpl.close(FileDispatcherImpl.java:127) 
> ~[na:1.8.0]
>       at 
> sun.nio.ch.FileChannelImpl.implCloseChannel(FileChannelImpl.java:153) 
> ~[na:1.8.0]
>       at 
> java.nio.channels.spi.AbstractInterruptibleChannel.close(AbstractInterruptibleChannel.java:126)
>  ~[na:1.8.0]
>       at java.nio.channels.Channels$1.close(Channels.java:189) ~[na:1.8.0]
>       at sun.nio.cs.StreamEncoder.implClose(StreamEncoder.java:332) 
> ~[na:1.8.0]
>       at sun.nio.cs.StreamEncoder.close(StreamEncoder.java:161) ~[na:1.8.0]
>       at java.io.OutputStreamWriter.close(OutputStreamWriter.java:295) 
> ~[na:1.8.0]
>       at java.io.BufferedWriter.close(BufferedWriter.java:277) ~[na:1.8.0]
>       at java.nio.file.Files.write(Files.java:3368) ~[na:1.8.0]
>       at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:581) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       ... 24 common frames omitted
> JVMPORT030W /proc/sys/kernel/core_pattern setting "|/usr/share/apport/apport 
> %p %s %c %d %P" specifies that the core dump is to be piped to an external 
> program.  Attempting to rename either core or core.1781.
> JVMDUMP012E Error in System dump: The core file created by child process with 
> pid = 1781 was not found. Expected to find core file with name "/tmp/core"
> JVMDUMP030W Cannot write dump to file //javacore.20180411.144944.84.0005.txt: 
> Permission denied
> JVMDUMP032I JVM requested Java dump using 
> '/tmp/javacore.20180411.144944.84.0005.txt' in response to an event
> JVMDUMP032I JVM requested Snap dump using 
> '//Snap.20180411.144942.84.0004.trc' in response to an event
> JVMDUMP030W Cannot write dump to file //Snap.20180411.144942.84.0004.trc: 
> Permission denied
> JVMDUMP010I Snap dump written to /tmp/Snap.20180411.144942.84.0004.trc
> JVMDUMP030W Cannot write dump to file //jitdump.20180411.144942.84.0006.dmp: 
> Permission denied
> JVMDUMP007I 
> #JITDUMP:  vmThread=0000000001B3B800 Recursive crash occurred. Aborting JIT 
> dump.JVMDUMP010I JIT dump written to /tmp/jitdump.20180411.144942.84.0006.dmp
> WARN  [MemtableFlushWriter:1] 2018-04-11 14:49:58,241 NativeLibrary.java:328 
> - fsync(3084) failed, errorno (22) {}
> com.sun.jna.LastErrorException: [22] Invalid argument
>       at org.apache.cassandra.utils.NativeLibraryLinux.fsync(Native Method) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.NativeLibraryLinux.callFsync(NativeLibraryLinux.java:107)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.NativeLibrary.trySync(NativeLibrary.java:317) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplica.syncDirectory(LogReplica.java:96)
>  [apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplica.delete(LogReplica.java:102) 
> [apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplicaSet$$Lambda$175.00000000BC451FC0.perform(Unknown
>  Source) ~[na:na]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogReplicaSet.delete(LogReplicaSet.java:92) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogFile.removeUnfinishedLeftovers(LogFile.java:128)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.run(LogTransaction.java:263)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.tidy(LogTransaction.java:242)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Ref$GlobalState.release(Ref.java:321) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Ref$State.ensureReleased(Ref.java:199) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Ref.ensureReleased(Ref.java:119) 
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogTransaction.complete(LogTransaction.java:377)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LogTransaction.doCommit(LogTransaction.java:389)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.lifecycle.LifecycleTransaction.doCommit(LifecycleTransaction.java:218)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.commit(Transactional.java:116)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1176)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1086)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160)
>  ~[na:1.8.0]
>       at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
>  ~[na:1.8.0]
>       at 
> org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81)
>  ~[apache-cassandra-3.11.0.jar:3.11.0]
>       at 
> org.apache.cassandra.concurrent.NamedThreadFactory$$Lambda$12.00000000E0F32EF0.run(Unknown
>  Source) ~[na:na]
>       at java.lang.Thread.run(Thread.java:811) ~[na:2.9 (12-15-2017)]
> JVMDUMP013I Processed dump event "gpf", detail "".
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org

Reply via email to