xiaobing0508 opened a new issue #12509:
URL: https://github.com/apache/pulsar/issues/12509
### In PULSAR_HOME/logs/pulsar-bookie.log :
16:00:28.189 [BookieReadThreadPool-OrderedExecutor-6-0] ERROR
org.apache.bookkeeper.proto.BookieServer - Unable to allocate memory, exiting
bookie
io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216
byte(s) of direct memory (used: 246960619520, max: 246960619520)
at
io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:802)
~[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:731)
~[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:632)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:126)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:395)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
org.apache.bookkeeper.bookie.storage.ldb.ReadCache.get(ReadCache.java:145)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.getEntry(DbLedgerStorage.java:200)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.LedgerDescriptorImpl.readEntry(LedgerDescriptorImpl.java:160)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at org.apache.bookkeeper.bookie.Bookie.readEntry(Bookie.java:1500)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.ReadEntryProcessor.processPacket(ReadEntryProcessor.java:83)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.PacketProcessorBase.safeRun(PacketProcessorBase.java:106)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
[org.apache.bookkeeper-bookkeeper-common-4.14.2.jar:4.14.2]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_181]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_181]
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216
byte(s) of direct memory (used: 42949672960, max: 42949672960)
at
io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:802)
~[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:731)
~[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:202)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.PoolArena.tcacheAllocateNormal(PoolArena.java:186)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:136)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:126)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:395)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
org.apache.bookkeeper.bookie.storage.ldb.ReadCache.get(ReadCache.java:145)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.getEntry(DbLedgerStorage.java:200)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.LedgerDescriptorImpl.readEntry(LedgerDescriptorImpl.java:160)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at org.apache.bookkeeper.bookie.Bookie.readEntry(Bookie.java:1500)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.ReadEntryProcessor.processPacket(ReadEntryProcessor.java:83)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.PacketProcessorBase.safeRun(PacketProcessorBase.java:106)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
[org.apache.bookkeeper-bookkeeper-common-4.14.2.jar:4.14.2]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_181]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_181]
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
16:00:28.190 [main-EventThread] INFO org.apache.zookeeper.ClientCnxn -
EventThread shut down for session: 0x200192f8cf60058
16:00:28.190 [BookieReadThreadPool-OrderedExecutor-6-0] ERROR
org.apache.bookkeeper.proto.ReadEntryProcessor - Unexpected exception reading
at 342:45964 : null
java.lang.IllegalThreadStateException: null
at java.lang.Thread.start(Thread.java:708) ~[?:1.8.0_181]
at
org.apache.bookkeeper.common.component.ComponentStarter.lambda$startComponent$0(ComponentStarter.java:79)
~[org.apache.bookkeeper-bookkeeper-common-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.BookieServer.lambda$getAllocator$1(BookieServer.java:322)
~[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorImpl.newDirectBuffer(ByteBufAllocatorImpl.java:174)
~[org.apache.bookkeeper-bookkeeper-common-allocator-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorImpl.newDirectBuffer(ByteBufAllocatorImpl.java:157)
~[org.apache.bookkeeper-bookkeeper-common-allocator-4.14.2.jar:4.14.2]
at
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
~[io.netty-netty-buffer-4.1.66.Final.jar:4.1.66.Final]
at
org.apache.bookkeeper.bookie.storage.ldb.ReadCache.get(ReadCache.java:145)
~[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.getEntry(SingleDirectoryDbLedgerStorage.java:438)
~[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.getEntry(DbLedgerStorage.java:200)
~[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.bookie.LedgerDescriptorImpl.readEntry(LedgerDescriptorImpl.java:160)
~[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at org.apache.bookkeeper.bookie.Bookie.readEntry(Bookie.java:1500)
~[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.ReadEntryProcessor.processPacket(ReadEntryProcessor.java:83)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.proto.PacketProcessorBase.safeRun(PacketProcessorBase.java:106)
[org.apache.bookkeeper-bookkeeper-server-4.14.2.jar:4.14.2]
at
org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
[org.apache.bookkeeper-bookkeeper-common-4.14.2.jar:4.14.2]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_181]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_181]
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
[io.netty-netty-common-4.1.66.Final.jar:4.1.66.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
BOOKIE_MEM MaxDirectMemorySize is 230GB.
can't set it to any number.
I don't know what happened.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
