lhotari commented on issue #19250: URL: https://github.com/apache/pulsar/issues/19250#issuecomment-1867917554
Yet another crash, this time in broker tests. ``` Stack: [0x00007f68f2c00000,0x00007f68f2d00000], sp=0x00007f68f2cfdac8, free space=1014k Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) C [librocksdbjni10411210078074458540.so+0x2adff0] rocksdb_iterator_helper(rocksdb::DB*, rocksdb::ReadOptions, rocksdb::ColumnFamilyHandle*)+0x60 j org.rocksdb.RocksDB.iterator(JJ)J+0 j org.rocksdb.RocksDB.newIterator(Lorg/rocksdb/ReadOptions;)Lorg/rocksdb/RocksIterator;+14 j org.apache.pulsar.metadata.impl.RocksdbMetadataStore.getChildrenFromStore(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+82 J 23070 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/String;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (21 bytes) @ 0x00007f6b5e554f6c [0x00007f6b5e554e40+0x000000000000012c] J 23069 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/Object;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (23 bytes) @ 0x00007f6b5df8bd44 [0x00007f6b5df8bbc0+0x0000000000000184] J 49063 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14([Ljava/lang/Object;Ljava/util/function/Function;Ljava/lang/Object;[J[I[Ljava/lang/Object;[Ljava/lang/Object;[Lcom/github/benmanes/caffeine/cache/RemovalCause;[Lcom/github/benmanes/caffeine/cache/Node;Ljava/lang/Object;Lcom/github/benmanes/caffeine/cache/Node;)Lcom/github/benmanes/caffeine/cache/Node; (355 bytes) @ 0x00007f6b66f75930 [0x00007f6b66f757a0+0x0000000000000190] J 28833 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda$502+0x00007f68f851a700.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object; (49 bytes) @ 0x00007f6b661163d8 [0x00007f6b661162c0+0x0000000000000118] J 50805 c1 java.util.concurrent.ConcurrentHashMap.compute(Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object; [email protected] (654 bytes) @ 0x00007f6b5e6bf814 [0x00007f6b5e6bf1e0+0x0000000000000634] J 50551 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(Ljava/lang/Object;Ljava/util/function/Function;ZZ)Ljava/lang/Object; (161 bytes) @ 0x00007f6b65f631e0 [0x00007f6b65f628e0+0x0000000000000900] J 26511 c2 com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture; (20 bytes) @ 0x00007f6b65e25680 [0x00007f6b65e25460+0x0000000000000220] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.getChildren(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+69 J 29539 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getBookiesThenFreshCache(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture; (91 bytes) @ 0x00007f6b5f7bf8cc [0x00007f6b5f7bf6c0+0x000000000000020c] j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getWritableBookies()Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.lambda$updatedBookies$26(Lorg/apache/pulsar/metadata/api/Notification;Lorg/apache/bookkeeper/net/BookieId;Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+355 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$4744+0x00007f68f903fd90.get()Ljava/lang/Object;+16 J 32154 c1 org.apache.pulsar.common.util.FutureUtil$Sequencer.sequential(Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture; (160 bytes) @ 0x00007f6b5f993f9c [0x00007f6b5f9932e0+0x0000000000000cbc] J 50820 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.updatedBookies(Lorg/apache/pulsar/metadata/api/Notification;)V (146 bytes) @ 0x00007f6b5e69f42c [0x00007f6b5e69ec40+0x00000000000007ec] J 50819 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$2017+0x00007f68f8be2c70.accept(Ljava/lang/Object;)V (12 bytes) @ 0x00007f6b5e6be6cc [0x00007f6b5e6be540+0x000000000000018c] J 33866 c2 java.util.concurrent.CompletableFuture$AsyncSupply.run()V [email protected] (61 bytes) @ 0x00007f6b65e60ea4 [0x00007f6b65e60d00+0x00000000000001a4] J 7662 c2 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x00007f6b6525bb84 [0x00007f6b6525bb20+0x0000000000000064] J 41512 c2 java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run()V [email protected] (57 bytes) @ 0x00007f6b658425e4 [0x00007f6b65842360+0x0000000000000284] J 23159 c2 java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V [email protected] (187 bytes) @ 0x00007f6b65b663cc [0x00007f6b65b66200+0x00000000000001cc] J 39308 c2 java.util.concurrent.ThreadPoolExecutor$Worker.run()V [email protected] (9 bytes) @ 0x00007f6b653e9000 [0x00007f6b653e8fa0+0x0000000000000060] J 30635 c2 io.netty.util.concurrent.FastThreadLocalRunnable.run()V (22 bytes) @ 0x00007f6b6633bdb8 [0x00007f6b6633bd40+0x0000000000000078] J 32703 c2 java.lang.Thread.run()V [email protected] (17 bytes) @ 0x00007f6b664f4488 [0x00007f6b664f4420+0x0000000000000068] v ~StubRoutines::call_stub V [libjvm.so+0x829765] JavaCalls::call_helper(JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*)+0x315 V [libjvm.so+0x82af5b] JavaCalls::call_virtual(JavaValue*, Handle, Klass*, Symbol*, Symbol*, JavaThread*)+0x1cb V [libjvm.so+0x8f7843] thread_entry(JavaThread*, JavaThread*)+0xa3 V [libjvm.so+0xe5ef76] JavaThread::thread_main_inner()+0x196 V [libjvm.so+0xe629b0] Thread::call_run()+0xc0 V [libjvm.so+0xc23e91] thread_native_entry(Thread*)+0xe1 Java frames: (J=compiled Java code, j=interpreted, Vv=VM code) j org.rocksdb.RocksDB.iterator(JJ)J+0 j org.rocksdb.RocksDB.newIterator(Lorg/rocksdb/ReadOptions;)Lorg/rocksdb/RocksIterator;+14 j org.apache.pulsar.metadata.impl.RocksdbMetadataStore.getChildrenFromStore(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+82 J 23070 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/String;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (21 bytes) @ 0x00007f6b5e554f6c [0x00007f6b5e554e40+0x000000000000012c] J 23069 c1 org.apache.pulsar.metadata.impl.AbstractMetadataStore$1.asyncLoad(Ljava/lang/Object;Ljava/util/concurrent/Executor;)Ljava/util/concurrent/CompletableFuture; (23 bytes) @ 0x00007f6b5df8bd44 [0x00007f6b5df8bbc0+0x0000000000000184] J 49063 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14([Ljava/lang/Object;Ljava/util/function/Function;Ljava/lang/Object;[J[I[Ljava/lang/Object;[Ljava/lang/Object;[Lcom/github/benmanes/caffeine/cache/RemovalCause;[Lcom/github/benmanes/caffeine/cache/Node;Ljava/lang/Object;Lcom/github/benmanes/caffeine/cache/Node;)Lcom/github/benmanes/caffeine/cache/Node; (355 bytes) @ 0x00007f6b66f75930 [0x00007f6b66f757a0+0x0000000000000190] J 28833 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda$502+0x00007f68f851a700.apply(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object; (49 bytes) @ 0x00007f6b661163d8 [0x00007f6b661162c0+0x0000000000000118] J 50805 c1 java.util.concurrent.ConcurrentHashMap.compute(Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object; [email protected] (654 bytes) @ 0x00007f6b5e6bf814 [0x00007f6b5e6bf1e0+0x0000000000000634] J 50551 c2 com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(Ljava/lang/Object;Ljava/util/function/Function;ZZ)Ljava/lang/Object; (161 bytes) @ 0x00007f6b65f631e0 [0x00007f6b65f628e0+0x0000000000000900] J 26511 c2 com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(Ljava/lang/Object;)Ljava/util/concurrent/CompletableFuture; (20 bytes) @ 0x00007f6b65e25680 [0x00007f6b65e25460+0x0000000000000220] j org.apache.pulsar.metadata.impl.AbstractMetadataStore.getChildren(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+69 J 29539 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getBookiesThenFreshCache(Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture; (91 bytes) @ 0x00007f6b5f7bf8cc [0x00007f6b5f7bf6c0+0x000000000000020c] j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.getWritableBookies()Ljava/util/concurrent/CompletableFuture;+12 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.lambda$updatedBookies$26(Lorg/apache/pulsar/metadata/api/Notification;Lorg/apache/bookkeeper/net/BookieId;Ljava/lang/String;)Ljava/util/concurrent/CompletableFuture;+355 j org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$4744+0x00007f68f903fd90.get()Ljava/lang/Object;+16 J 32154 c1 org.apache.pulsar.common.util.FutureUtil$Sequencer.sequential(Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture; (160 bytes) @ 0x00007f6b5f993f9c [0x00007f6b5f9932e0+0x0000000000000cbc] J 50820 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient.updatedBookies(Lorg/apache/pulsar/metadata/api/Notification;)V (146 bytes) @ 0x00007f6b5e69f42c [0x00007f6b5e69ec40+0x00000000000007ec] J 50819 c1 org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient$$Lambda$2017+0x00007f68f8be2c70.accept(Ljava/lang/Object;)V (12 bytes) @ 0x00007f6b5e6be6cc [0x00007f6b5e6be540+0x000000000000018c] J 33866 c2 java.util.concurrent.CompletableFuture$AsyncSupply.run()V [email protected] (61 bytes) @ 0x00007f6b65e60ea4 [0x00007f6b65e60d00+0x00000000000001a4] J 7662 c2 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x00007f6b6525bb84 [0x00007f6b6525bb20+0x0000000000000064] J 41512 c2 java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run()V [email protected] (57 bytes) @ 0x00007f6b658425e4 [0x00007f6b65842360+0x0000000000000284] J 23159 c2 java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V [email protected] (187 bytes) @ 0x00007f6b65b663cc [0x00007f6b65b66200+0x00000000000001cc] J 39308 c2 java.util.concurrent.ThreadPoolExecutor$Worker.run()V [email protected] (9 bytes) @ 0x00007f6b653e9000 [0x00007f6b653e8fa0+0x0000000000000060] J 30635 c2 io.netty.util.concurrent.FastThreadLocalRunnable.run()V (22 bytes) @ 0x00007f6b6633bdb8 [0x00007f6b6633bd40+0x0000000000000078] J 32703 c2 java.lang.Thread.run()V [email protected] (17 bytes) @ 0x00007f6b664f4488 [0x00007f6b664f4420+0x0000000000000068] v ~StubRoutines::call_stub ``` full dump https://gist.github.com/lhotari/f35745b737e7107f13c8becea80f9da4 location: https://github.com/apache/pulsar/actions/runs/7299672614/job/19893235168#step:11:1590 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
