lhotari commented on issue #19250: URL: https://github.com/apache/pulsar/issues/19250#issuecomment-1789742057
Recent crash: https://gist.github.com/lhotari/6c66d0dc55a5104d4364cdbafe26d2e9 ``` Stack: [0x00007fe85af00000,0x00007fe85b000000], sp=0x00007fe85affd4b0, free space=1013k Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) C [librocksdbjni6450051497838230364.so+0x2aece0] Java_org_rocksdb_RocksDB_iterator__JJ+0x30 j org.rocksdb.RocksDB.iterator(JJ)J+0 j org.rocksdb.RocksDB.newIterator(Lorg/rocksdb/ReadOptions;)Lorg/rocksdb/RocksIterator;+14 j org.apache.pulsar.metadata.impl.RocksdbMetadataStore.getChildrenFromStore(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+82 j org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/String;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/Object;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture;+13 j com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache$$Lambda$373+0x00007fe874410fa0.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+9 j com.github.benmanes.caffeine.cache.LocalAsyncCache.lambda$get$2([Ljava/util/concurrent/CompletableFuture;Ljava/util/function/BiFunction;Ljava/lang/Object;Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture;+15 j com.github.benmanes.caffeine.cache.LocalAsyncCache$$Lambda$374+0x00007fe874411630.apply(Ljava/lang/Object;)Ljava/lang/Object;+17 j com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14([Ljava/lang/Object;Ljava/util/function/Function;Ljava/lang/Object;[J[I[Ljava/lang/Object;[Ljava/lang/Object;[Lcom/github/benmanes/caffeine/cache/RemovalCause;[Lcom/github/benmanes/caffeine/cache/Node;Ljava/lang/Object;Lcom/github/benmanes/caffeine/cache/Node;)Lcom/github/benmanes/caffeine/cache/Node;+9 j com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda$375+0x00007fe8744122a8.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+45 J 9835 c1 java.util.concurrent.ConcurrentHashMap.compute(Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object; [email protected] (654 bytes) @ 0x00007feada176494 [0x00007feada175e60+0x0000000000000634] j com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(Ljava/lang/Object;Ljava/lang/Object;Ljava/util/function/Function;[JZ)Ljava/lang/Object;+62 j com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(Ljava/lang/Object;Ljava/util/function/Function;ZZ)Ljava/lang/Object;+157 j com.github.benmanes.caffeine.cache.LocalAsyncCache.get(Ljava/lang/Object;Ljava/util/function/BiFunction;Z)Ljava/util/concurrent/CompletableFuture;+43 J 10126 c1 com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture; (20 bytes) @ 0x00007feada20d98c [0x00007feada20d340+0x000000000000064c] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.getChildren(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+69 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getBookiesThenFreshCache(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+54 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getWritableBookies()Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.lambda$updatedBookies$26(Lorg/apache/pulsar/metadata/api/Notification;Lorg/apache/bookkeeper/net/BookieId;Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+355 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$445+0x00007fe8745008d0.get()Ljava/lang/Object;+16 j org.apache.pulsar.common.util.FutureUtil$Sequencer.sequential(Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture;+72 J 9837 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.updatedBookies(Lorg/apache/pulsar/metadata/api/Notification;)V (146 bytes) @ 0x00007feada1815e4 [0x00007feada1812c0+0x0000000000000324] J 8862 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$395+0x00007fe87442c230.accept(Ljava/lang/Object;)V (12 bytes) @ 0x00007fead96805f4 [0x00007fead9680540+0x00000000000000b4] J 8829 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore.lambda$receivedNotification$5(Lorg/apache/pulsar/metadata/api/Notification;Ljava/util/function/Consumer;)V (57 bytes) @ 0x00007fead99e29e4 [0x00007fead99e28e0+0x0000000000000104] J 8828 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$$Lambda$381+0x00007fe874419180.accept(Ljava/lang/Object;)V (12 bytes) @ 0x00007fead9a202e4 [0x00007fead9a20140+0x00000000000001a4] J 8617 c1 java.util.concurrent.CopyOnWriteArrayList.forEach(Ljava/util/function/Consumer;)V [email protected] (47 bytes) @ 0x00007fead98d62ac [0x00007fead98d61e0+0x00000000000000cc] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.lambda$receivedNotification$6(Lorg/apache/pulsar/metadata/api/Notification;)Ljava/lang/Void;+17 j org.apache.pulsar.metadata.impl.AbstractMetadataStore$$Lambda$378+0x00007fe8744181f8.get()Ljava/lang/Object;+8 j java.util.concurrent.CompletableFuture$AsyncSupply.run()V+37 [email protected] J 6908 c2 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x00007feae1156104 [0x00007feae11560a0+0x0000000000000064] J 7466 c1 java.util.concurrent.FutureTask.run()V [email protected] (123 bytes) @ 0x00007fead9f7da2c [0x00007fead9f7d220+0x000000000000080c] J 6998 c1 java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run()V [email protected] (57 bytes) @ 0x00007fead9e8850c [0x00007fead9e88140+0x00000000000003cc] j java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V+92 [email protected] j java.util.concurrent.ThreadPoolExecutor$Worker.run()V+5 [email protected] J 10156 c1 io.netty.util.concurrent.FastThreadLocalRunnable.run()V (22 bytes) @ 0x00007feada21c90c [0x00007feada21c7e0+0x000000000000012c] J 9973 c1 java.lang.Thread.run()V [email protected] (17 bytes) @ 0x00007feada1b1624 [0x00007feada1b14c0+0x0000000000000164] v ~StubRoutines::call_stub V [libjvm.so+0x827355] JavaCalls::call_helper(JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*)+0x315 V [libjvm.so+0x828b4b] JavaCalls::call_virtual(JavaValue*, Handle, Klass*, Symbol*, Symbol*, JavaThread*)+0x1cb V [libjvm.so+0x8f3ae3] thread_entry(JavaThread*, JavaThread*)+0xa3 V [libjvm.so+0xe65584] JavaThread::thread_main_inner()+0x184 V [libjvm.so+0xe68d40] Thread::call_run()+0xc0 V [libjvm.so+0xc1e681] thread_native_entry(Thread*)+0xe1 Java frames: (J=compiled Java code, j=interpreted, Vv=VM code) j org.rocksdb.RocksDB.iterator(JJ)J+0 j org.rocksdb.RocksDB.newIterator(Lorg/rocksdb/ReadOptions;)Lorg/rocksdb/RocksIterator;+14 j org.apache.pulsar.metadata.impl.RocksdbMetadataStore.getChildrenFromStore(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+82 j org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/String;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/Object;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture;+13 j com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache$$Lambda$373+0x00007fe874410fa0.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+9 j com.github.benmanes.caffeine.cache.LocalAsyncCache.lambda$get$2([Ljava/util/concurrent/CompletableFuture;Ljava/util/function/BiFunction;Ljava/lang/Object;Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture;+15 j com.github.benmanes.caffeine.cache.LocalAsyncCache$$Lambda$374+0x00007fe874411630.apply(Ljava/lang/Object;)Ljava/lang/Object;+17 j com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14([Ljava/lang/Object;Ljava/util/function/Function;Ljava/lang/Object;[J[I[Ljava/lang/Object;[Ljava/lang/Object;[Lcom/github/benmanes/caffeine/cache/RemovalCause;[Lcom/github/benmanes/caffeine/cache/Node;Ljava/lang/Object;Lcom/github/benmanes/caffeine/cache/Node;)Lcom/github/benmanes/caffeine/cache/Node;+9 j com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda$375+0x00007fe8744122a8.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+45 J 9835 c1 java.util.concurrent.ConcurrentHashMap.compute(Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object; [email protected] (654 bytes) @ 0x00007feada176494 [0x00007feada175e60+0x0000000000000634] j com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(Ljava/lang/Object;Ljava/lang/Object;Ljava/util/function/Function;[JZ)Ljava/lang/Object;+62 j com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(Ljava/lang/Object;Ljava/util/function/Function;ZZ)Ljava/lang/Object;+157 j com.github.benmanes.caffeine.cache.LocalAsyncCache.get(Ljava/lang/Object;Ljava/util/function/BiFunction;Z)Ljava/util/concurrent/CompletableFuture;+43 J 10126 c1 com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture; (20 bytes) @ 0x00007feada20d98c [0x00007feada20d340+0x000000000000064c] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.getChildren(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+69 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getBookiesThenFreshCache(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+54 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getWritableBookies()Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.lambda$updatedBookies$26(Lorg/apache/pulsar/metadata/api/Notification;Lorg/apache/bookkeeper/net/BookieId;Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+355 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$445+0x00007fe8745008d0.get()Ljava/lang/Object;+16 j org.apache.pulsar.common.util.FutureUtil$Sequencer.sequential(Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture;+72 J 9837 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.updatedBookies(Lorg/apache/pulsar/metadata/api/Notification;)V (146 bytes) @ 0x00007feada1815e4 [0x00007feada1812c0+0x0000000000000324] J 8862 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$395+0x00007fe87442c230.accept(Ljava/lang/Object;)V (12 bytes) @ 0x00007fead96805f4 [0x00007fead9680540+0x00000000000000b4] J 8829 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore.lambda$receivedNotification$5(Lorg/apache/pulsar/metadata/api/Notification;Ljava/util/function/Consumer;)V (57 bytes) @ 0x00007fead99e29e4 [0x00007fead99e28e0+0x0000000000000104] J 8828 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$$Lambda$381+0x00007fe874419180.accept(Ljava/lang/Object;)V (12 bytes) @ 0x00007fead9a202e4 [0x00007fead9a20140+0x00000000000001a4] J 8617 c1 java.util.concurrent.CopyOnWriteArrayList.forEach(Ljava/util/function/Consumer;)V [email protected] (47 bytes) @ 0x00007fead98d62ac [0x00007fead98d61e0+0x00000000000000cc] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.lambda$receivedNotification$6(Lorg/apache/pulsar/metadata/api/Notification;)Ljava/lang/Void;+17 j org.apache.pulsar.metadata.impl.AbstractMetadataStore$$Lambda$378+0x00007fe8744181f8.get()Ljava/lang/Object;+8 j java.util.concurrent.CompletableFuture$AsyncSupply.run()V+37 [email protected] J 6908 c2 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x00007feae1156104 [0x00007feae11560a0+0x0000000000000064] J 7466 c1 java.util.concurrent.FutureTask.run()V [email protected] (123 bytes) @ 0x00007fead9f7da2c [0x00007fead9f7d220+0x000000000000080c] J 6998 c1 java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run()V [email protected] (57 bytes) @ 0x00007fead9e8850c [0x00007fead9e88140+0x00000000000003cc] j java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V+92 [email protected] j java.util.concurrent.ThreadPoolExecutor$Worker.run()V+5 [email protected] J 10156 c1 io.netty.util.concurrent.FastThreadLocalRunnable.run()V (22 bytes) @ 0x00007feada21c90c [0x00007feada21c7e0+0x000000000000012c] J 9973 c1 java.lang.Thread.run()V [email protected] (17 bytes) @ 0x00007feada1b1624 [0x00007feada1b14c0+0x0000000000000164] v ~StubRoutines::call_stub ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
