lhotari commented on issue #19250: URL: https://github.com/apache/pulsar/issues/19250#issuecomment-2006591170
Another recent crash: https://gist.github.com/lhotari/c95a1bbf42880d49b0de508f7870f94a ``` Stack: [0x0000720a49200000,0x0000720a49300000], sp=0x0000720a492fdab8, free space=1014k Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) C [librocksdbjni10433916121165253334.so+0x2adff0] rocksdb_iterator_helper(rocksdb::DB*, rocksdb::ReadOptions, rocksdb::ColumnFamilyHandle*)+0x60 j org.rocksdb.RocksDB.iterator(JJ)J+0 j org.rocksdb.RocksDB.newIterator(Lorg/rocksdb/ReadOptions;)Lorg/rocksdb/RocksIterator;+14 j org.apache.pulsar.metadata.impl.RocksdbMetadataStore.getChildrenFromStore(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+82 J 53586 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/String;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (21 bytes) @ 0x0000720c6ab9d16c [0x0000720c6ab9d040+0x000000000000012c] J 50411 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/Object;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (23 bytes) @ 0x0000720c6962c6b4 [0x0000720c6962c540+0x0000000000000174] J 39087 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14([Ljava/lang/Object;Ljava/util/function/Function;Ljava/lang/Object;[J[I[Ljava/lang/Object;[Ljava/lang/Object;[Lcom/github/benmanes/caffeine/cache/RemovalCause;[Lcom/github/benmanes/caffeine/cache/Node;Ljava/lang/Object;Lcom/github/benmanes/caffeine/cache/Node;)Lcom/github/benmanes/caffeine/cache/Node; (355 bytes) @ 0x0000720c72741934 [0x0000720c72741780+0x00000000000001b4] J 42550 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda$496+0x00007209fc4f6640.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object; (49 bytes) @ 0x0000720c729bdcd8 [0x0000720c729bdbc0+0x0000000000000118] J 55012 c2 java.util.concurrent.ConcurrentHashMap.compute(Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object; [email protected] (822 bytes) @ 0x0000720c72ea1880 [0x0000720c72ea0980+0x0000000000000f00] J 33464 c1 com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(Ljava/lang/Object;Ljava/lang/Object;Ljava/util/function/Function;[JZ)Ljava/lang/Object; (276 bytes) @ 0x0000720c6b78e49c [0x0000720c6b78dc00+0x000000000000089c] j com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(Ljava/lang/Object;Ljava/util/function/Function;ZZ)Ljava/lang/Object;+157 J 39494 c2 com.github.benmanes.caffeine.cache.LocalAsyncCache.get(Ljava/lang/Object;Ljava/util/function/BiFunction;Z)Ljava/util/concurrent/CompletableFuture; (77 bytes) @ 0x0000720c727e6fb4 [0x0000720c727e6e00+0x00000000000001b4] J 21863 c2 com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture; (20 bytes) @ 0x0000720c718feb54 [0x0000720c718feaa0+0x00000000000000b4] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.getChildren(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+178 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getBookiesThenFreshCache(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+54 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getWritableBookies()Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.lambda$updatedBookies$26(Lorg/apache/pulsar/metadata/api/Notification;Lorg/apache/bookkeeper/net/BookieId;Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+355 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$4937+0x00007209fd15bd40.get()Ljava/lang/Object;+16 J 36324 c1 org.apache.pulsar.common.util.FutureUtil$Sequencer.sequential(Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture; (160 bytes) @ 0x0000720c6ba8f9f4 [0x0000720c6ba8f040+0x00000000000009b4] J 56975 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.updatedBookies(Lorg/apache/pulsar/metadata/api/Notification;)V (146 bytes) @ 0x0000720c6d50ace4 [0x0000720c6d50a780+0x0000000000000564] J 56393 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$2075+0x00007209fcb96f08.accept(Ljava/lang/Object;)V (12 bytes) @ 0x0000720c6d39254c [0x0000720c6d3923c0+0x000000000000018c] J 53585 c2 java.util.concurrent.CompletableFuture$AsyncSupply.run()V [email protected] (61 bytes) @ 0x0000720c71643ddc [0x0000720c716439c0+0x000000000000041c] J 8303 c2 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x0000720c7128db04 [0x0000720c7128daa0+0x0000000000000064] J 34960 c2 java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run()V [email protected] (57 bytes) @ 0x0000720c723c7f60 [0x0000720c723c7ce0+0x0000000000000280] J 32343 c2 java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V [email protected] (187 bytes) @ 0x0000720c71b23694 [0x0000720c71b23440+0x0000000000000254] J 51463 c2 java.util.concurrent.ThreadPoolExecutor$Worker.run()V [email protected] (9 bytes) @ 0x0000720c712ab280 [0x0000720c712ab220+0x0000000000000060] J 48582 c2 io.netty.util.concurrent.FastThreadLocalRunnable.run()V (22 bytes) @ 0x0000720c73011438 [0x0000720c730113c0+0x0000000000000078] J 43180 c2 java.lang.Thread.run()V [email protected] (17 bytes) @ 0x0000720c72a80588 [0x0000720c72a80520+0x0000000000000068] v ~StubRoutines::call_stub V [libjvm.so+0x859824] JavaCalls::call_helper(JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*)+0x334 V [libjvm.so+0x85b30c] JavaCalls::call_virtual(JavaValue*, Handle, Klass*, Symbol*, Symbol*, JavaThread*)+0x20c V [libjvm.so+0x919760] thread_entry(JavaThread*, JavaThread*)+0x70 V [libjvm.so+0xed9a47] JavaThread::run()+0x127 V [libjvm.so+0xedcbcf] Thread::call_run()+0x9f V [libjvm.so+0xc4ebb1] thread_native_entry(Thread*)+0xe1 Java frames: (J=compiled Java code, j=interpreted, Vv=VM code) j org.rocksdb.RocksDB.iterator(JJ)J+0 j org.rocksdb.RocksDB.newIterator(Lorg/rocksdb/ReadOptions;)Lorg/rocksdb/RocksIterator;+14 j org.apache.pulsar.metadata.impl.RocksdbMetadataStore.getChildrenFromStore(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+82 J 53586 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/String;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (21 bytes) @ 0x0000720c6ab9d16c [0x0000720c6ab9d040+0x000000000000012c] J 50411 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/Object;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (23 bytes) @ 0x0000720c6962c6b4 [0x0000720c6962c540+0x0000000000000174] J 39087 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14([Ljava/lang/Object;Ljava/util/function/Function;Ljava/lang/Object;[J[I[Ljava/lang/Object;[Ljava/lang/Object;[Lcom/github/benmanes/caffeine/cache/RemovalCause;[Lcom/github/benmanes/caffeine/cache/Node;Ljava/lang/Object;Lcom/github/benmanes/caffeine/cache/Node;)Lcom/github/benmanes/caffeine/cache/Node; (355 bytes) @ 0x0000720c72741934 [0x0000720c72741780+0x00000000000001b4] J 42550 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda$496+0x00007209fc4f6640.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object; (49 bytes) @ 0x0000720c729bdcd8 [0x0000720c729bdbc0+0x0000000000000118] J 55012 c2 java.util.concurrent.ConcurrentHashMap.compute(Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object; [email protected] (822 bytes) @ 0x0000720c72ea1880 [0x0000720c72ea0980+0x0000000000000f00] J 33464 c1 com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(Ljava/lang/Object;Ljava/lang/Object;Ljava/util/function/Function;[JZ)Ljava/lang/Object; (276 bytes) @ 0x0000720c6b78e49c [0x0000720c6b78dc00+0x000000000000089c] j com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(Ljava/lang/Object;Ljava/util/function/Function;ZZ)Ljava/lang/Object;+157 J 39494 c2 com.github.benmanes.caffeine.cache.LocalAsyncCache.get(Ljava/lang/Object;Ljava/util/function/BiFunction;Z)Ljava/util/concurrent/CompletableFuture; (77 bytes) @ 0x0000720c727e6fb4 [0x0000720c727e6e00+0x00000000000001b4] J 21863 c2 com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture; (20 bytes) @ 0x0000720c718feb54 [0x0000720c718feaa0+0x00000000000000b4] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.getChildren(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+178 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getBookiesThenFreshCache(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+54 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getWritableBookies()Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.lambda$updatedBookies$26(Lorg/apache/pulsar/metadata/api/Notification;Lorg/apache/bookkeeper/net/BookieId;Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+355 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$4937+0x00007209fd15bd40.get()Ljava/lang/Object;+16 J 36324 c1 org.apache.pulsar.common.util.FutureUtil$Sequencer.sequential(Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture; (160 bytes) @ 0x0000720c6ba8f9f4 [0x0000720c6ba8f040+0x00000000000009b4] J 56975 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.updatedBookies(Lorg/apache/pulsar/metadata/api/Notification;)V (146 bytes) @ 0x0000720c6d50ace4 [0x0000720c6d50a780+0x0000000000000564] J 56393 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$2075+0x00007209fcb96f08.accept(Ljava/lang/Object;)V (12 bytes) @ 0x0000720c6d39254c [0x0000720c6d3923c0+0x000000000000018c] J 53585 c2 java.util.concurrent.CompletableFuture$AsyncSupply.run()V [email protected] (61 bytes) @ 0x0000720c71643ddc [0x0000720c716439c0+0x000000000000041c] J 8303 c2 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x0000720c7128db04 [0x0000720c7128daa0+0x0000000000000064] J 34960 c2 java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run()V [email protected] (57 bytes) @ 0x0000720c723c7f60 [0x0000720c723c7ce0+0x0000000000000280] J 32343 c2 java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V [email protected] (187 bytes) @ 0x0000720c71b23694 [0x0000720c71b23440+0x0000000000000254] J 51463 c2 java.util.concurrent.ThreadPoolExecutor$Worker.run()V [email protected] (9 bytes) @ 0x0000720c712ab280 [0x0000720c712ab220+0x0000000000000060] J 48582 c2 io.netty.util.concurrent.FastThreadLocalRunnable.run()V (22 bytes) @ 0x0000720c73011438 [0x0000720c730113c0+0x0000000000000078] J 43180 c2 java.lang.Thread.run()V [email protected] (17 bytes) @ 0x0000720c72a80588 [0x0000720c72a80520+0x0000000000000068] v ~StubRoutines::call_stub ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
