[ https://issues.apache.org/jira/browse/SOLR-10951?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16062375#comment-16062375 ]
Uwe Schindler commented on SOLR-10951: -------------------------------------- I had to change some additional dependencies, but I got it running up to the place where tests of HDFS tear down. With Hadoop-2.8.1, the test shutdown code and thread leak detector hangs indefinite in a locking while trying to interrupt() threads: {noformat} [junit4] >>> JVM J1 emitted unexpected output (verbatim) ---- [junit4] 2017-06-25 15:58:44 [junit4] Full thread dump Java HotSpot(TM) Client VM (25.131-b11 mixed mode): [junit4] [junit4] "IPC Server handler 9 on 41067" #8132 daemon prio=5 os_prio=0 tid=0xc6674800 nid=0x67c3 waiting on condition [0xca932000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 8 on 41067" #8131 daemon prio=5 os_prio=0 tid=0xc6673c00 nid=0x67c2 waiting on condition [0xcf2ad000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 7 on 41067" #8130 daemon prio=5 os_prio=0 tid=0xc6672c00 nid=0x67c1 waiting on condition [0xcf25c000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awa [junit4] itNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 6 on 41067" #8129 daemon prio=5 os_prio=0 tid=0xc6671c00 nid=0x67c0 waiting on condition [0xca515000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 5 on 41067" #8128 daemon prio=5 os_prio=0 tid=0xc6670c00 nid=0x67bf waiting on condition [0xcaa76000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 4 on 41067" #8127 daemon prio=5 os_prio=0 tid=0xc6670000 nid=0x67be waiting on condition [0xca5b7000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchro [junit4] nizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 3 on 41067" #8126 daemon prio=5 os_prio=0 tid=0xc666f000 nid=0x67bd waiting on condition [0xca3d1000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 2 on 41067" #8125 daemon prio=5 os_prio=0 tid=0xc666e400 nid=0x67bc waiting on condition [0xce4ad000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 1 on 41067" #8124 daemon prio=5 os_prio=0 tid=0xc666d400 nid=0x67bb waiting on condition [0xca473000] [junit4] j [junit4] ava.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server handler 0 on 41067" #8123 daemon prio=5 os_prio=0 tid=0xc666c400 nid=0x67ba waiting on condition [0xc9ffe000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) [junit4] at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214) [junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441) [junit4] [junit4] "IPC Server listener on 41067" #8114 daemon prio=5 os_prio=0 tid=0xc666bc00 nid=0x67b9 runnable [0xc7018000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] at sun.nio.ch.IOUtil.drain(Native Method) [junit4] at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) [junit4] - locked <0xd7ce8eb0> (a java.lang.Object) [junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] - locked <0xd7ce8c70> (a sun.nio.ch.Util$3) [junit4] - locked <0xd7ce8c60> (a java.util.Collections$UnmodifiableSet) [junit4] - locked <0xd7ce8b28> (a sun.nio.ch.EPollSelectorImpl) [junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] at org.apache.hadoop.ipc.Server$Listener.r [junit4] un(Server.java:1036) [junit4] [junit4] "IPC Server Responder" #8117 daemon prio=5 os_prio=0 tid=0xc666b400 nid=0x67b8 runnable [0xc8f8a000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) [junit4] at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) [junit4] at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) [junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] - locked <0xd7b3e3e8> (a sun.nio.ch.Util$3) [junit4] - locked <0xd7b3e3f8> (a java.util.Collections$UnmodifiableSet) [junit4] - locked <0xd7b3e398> (a sun.nio.ch.EPollSelectorImpl) [junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1214) [junit4] at org.apache.hadoop.ipc.Server$Responder.run(Server.java:1197) [junit4] [junit4] "Block report processor" #8112 daemon prio=5 os_prio=0 tid=0xc6662800 nid=0x67b7 waiting on condition [0xca1eb000] [junit4] java.lang.Thread.State: WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) [junit4] at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:3984) [junit4] at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:3973) [junit4] [junit4] "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@1e1bd5d" #8113 daemon prio=5 os_prio=0 tid=0xc665ec00 nid=0x67b5 waiting on condition [0xc70ba000] [junit4] java.lang.Thread.State: TIMED_WAITING (sleeping) [junit4] at java.lang.Thread.sleep(Native Method) [junit4] at org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:419) [junit4] at java.lang.Thread.run(Thread.java:748) [junit4] [junit4] "DecommissionMonitor-0" #8122 d [junit4] aemon prio=5 os_prio=0 tid=0xc665e000 nid=0x67b4 waiting on condition [0xcf1ba000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3e9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] at java.lang.Thread.run(Thread.java:748) [junit4] [junit4] "org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@1964584" #8121 daemon prio=5 os_prio=0 tid=0xc665c400 nid=0x67b3 waiting on condition [0xc8df5000] [junit4] java.lang.Thread.State: TIMED_WAITING (sleeping) [junit4] at java.lang.Thread.sleep(Native Method) [junit4] at org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:237) [junit4] at java.lang.Thread.run(Thread.java:748) [junit4] [junit4] "IPC Server idle connection scanner for port 41067" #8116 daemon prio=5 os_prio=0 tid=0xc661b400 nid=0x67ad in Object.wait() [0xcb2fe000] [junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.util.TimerThread.mainLoop(Timer.java:552) [junit4] - locked <0xd7b3ed88> (a java.util.TaskQueue) [junit4] at java.util.TimerThread.run(Timer.java:505) [junit4] [junit4] "Socket Reader #1 for port 41067" #8115 prio=5 os_prio=0 tid=0xc6619400 nid=0x67ac runnable [0xc9c32000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] at sun.nio.ch.EPollArrayWrapper.interrupt(Na [junit4] tive Method) [junit4] at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) [junit4] at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) [junit4] - locked <0xd7b3ef60> (a java.lang.Object) [junit4] at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) [junit4] at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) [junit4] at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) [junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] - locked <0xd7b3ef68> (a sun.nio.ch.Util$3) [junit4] - locked <0xd7b3ef78> (a java.util.Collections$UnmodifiableSet) [junit4] - locked <0xd7b3ef20> (a sun.nio.ch.EPollSelectorImpl) [junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:974) [junit4] - locked <0xd7b3ed98> (a org.apache.hadoop.ipc.Server$Listener$Reader) [junit4] at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:953) [junit4] [junit4] "Timer-1" #8110 daemon prio=5 os_prio=0 tid=0xc7626400 nid=0x67ab in Object.wait() [0xcb069000] [junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.util.TimerThread.mainLoop(Timer.java:552) [junit4] - locked <0xd7b3f108> (a java.util.TaskQueue) [junit4] at java.util.TimerThread.run(Timer.java:505) [junit4] [junit4] "Timer-0" #8109 daemon prio=5 os_prio=0 tid=0xc760e000 nid=0x67aa in Object.wait() [0xc8f39000] [junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.util.TimerThread.mainLoop(Timer.java:552) [junit4] - locked <0xd7b3f2c8> (a java.util.TaskQueue) [junit4] at java.util.TimerThread.run(Timer.java:505) [junit4] [junit4] "13142186@qtp-24418277-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost.localdomain:43319" #8107 daemon prio=5 os_prio=0 tid=0xc7603800 nid=0x67a8 runnable [0xcaac7000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) [junit4] at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) [junit4] at sun.nio.ch.EPol [junit4] lSelectorImpl.doSelect(EPollSelectorImpl.java:93) [junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] - locked <0xd7e0b980> (a sun.nio.ch.Util$3) [junit4] - locked <0xd7e0b990> (a java.util.Collections$UnmodifiableSet) [junit4] - locked <0xd7e0b940> (a sun.nio.ch.EPollSelectorImpl) [junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) [junit4] at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) [junit4] at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) [junit4] at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) [junit4] at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) [junit4] [junit4] "pool-2-thread-1" #8105 prio=5 os_prio=0 tid=0xc7a73400 nid=0x67a6 waiting on condition [0xce1c7000] [junit4] java.lang.Thread.State: TIMED_WAITING (parking) [junit4] at sun.misc.Unsafe.park(Native Method) [junit4] - parking to wait for <0xd7b3f7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) [junit4] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] at java.lang.Thread.run(Thread.java:748) [junit4] [junit4] "Timer for 'NameNode' metrics system" #8102 daemon prio=5 os_prio=0 tid=0xc7a52000 nid=0x67a4 in Object.wait() [0xc9b3f000] [junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.util.TimerThread.mainLoop(Timer.java:552) [junit4] - locked <0xd7b3fa38> (a java.util.TaskQueue) [junit4] at java.util.TimerThread.run(Timer.java:505) [junit4] [junit4] "SUITE-HdfsDirectoryFactoryTest-seed#[61471115528599BC]" #8093 prio=10 os_prio=0 tid=0xcaec4400 nid=0x679d waiting for monitor entry [0xcc159000] [junit4] java.lang.Thread.State: BLOCKED (on object monitor) [junit4] at java.lang.Thread.join(Thread.java:1243) [junit4] - waiting to lock <0xd7b3ed98> (a org.apache.hadoop.ipc.Server$Listener$Reader) [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl.tryToInterruptAll(ThreadLeakControl.java:776) [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl.checkThreadLeaks(ThreadLeakControl.java:626) [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:439) [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678) [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140) [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598) [junit4] [junit4] "TimeLimitedCollector timer thread" #2531 daemon prio=5 os_prio=0 tid=0xf2cd9800 nid=0x3d98 waiting on condition [0xc96d1000] [junit4] java.lang.Thread.State: TIMED_WAITING (sleeping) [junit4] at java.lang.Thread.sleep(Native Method) [junit4] at org.apache.lucene.search.TimeLimitingCollector$TimerThread.run(TimeLimitingCollector.java:261) [junit4] [junit4] "JUnit4-serializer-daemon" #8 daemon prio=5 os_prio=0 tid=0xd027cc00 nid=0x2266 waiting on condition [0xd147b000] [junit4] java.lang.Thread.State: TIMED_WAITING (sleeping) [junit4] at java.lang.Thread.sleep(Native Method) [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50) [junit4] [junit4] "Service Thread" #7 daemon prio=9 os_prio=0 tid=0xd0a2a400 nid=0x2261 runnable [0x00000000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] [junit4] "C1 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0xd0a26c00 nid=0x225f waiting on condition [0x00000000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] [junit4] "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0xd0a25400 nid=0x225e waiting on condition [0x00000000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] [junit4] "Surrogate Locker Thre [junit4] ad (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0xd0a23c00 nid=0x225d waiting on condition [0x00000000] [junit4] java.lang.Thread.State: RUNNABLE [junit4] [junit4] "Finalizer" #3 daemon prio=8 os_prio=0 tid=0xd13d9400 nid=0x2254 in Object.wait() [0xf3b7d000] [junit4] java.lang.Thread.State: WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) [junit4] - locked <0xd301c770> (a java.lang.ref.ReferenceQueue$Lock) [junit4] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) [junit4] at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) [junit4] [junit4] "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0xd13d6800 nid=0x2253 in Object.wait() [0xf3d7d000] [junit4] java.lang.Thread.State: WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.lang.Object.wait(Object.java:502) [junit4] at java.lang.ref.Reference.tryHandlePending(Reference.java:191) [junit4] - locked <0xd301c910> (a java.lang.ref.Reference$Lock) [junit4] at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) [junit4] [junit4] "main" #1 prio=5 os_prio=0 tid=0xf6912c00 nid=0x2214 in Object.wait() [0xf6a65000] [junit4] java.lang.Thread.State: WAITING (on object monitor) [junit4] at java.lang.Object.wait(Native Method) [junit4] at java.lang.Thread.join(Thread.java:1252) [junit4] - locked <0xd7b3fa78> (a com.carrotsearch.randomizedtesting.RandomizedRunner$2) [junit4] at java.lang.Thread.join(Thread.java:1326) [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608) [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457) [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:244) [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355) [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13) [junit4] [junit4] "VM Thread" os_prio=0 tid=0xd13d1c00 nid=0x2252 runnable [junit4] [junit4] "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0xf691d800 nid=0x2229 runnable [junit4] [junit4] "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0xf691ec00 nid=0x222c runnable [junit4] [junit4] "Gang worker#2 [junit4] (Parallel GC Threads)" os_prio=0 tid=0xf6920000 nid=0x222f runnable [junit4] [junit4] "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0xf6921400 nid=0x2232 runnable [junit4] [junit4] "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0xf6922c00 nid=0x2235 runnable [junit4] [junit4] "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0xf6924000 nid=0x2238 runnable [junit4] [junit4] "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0xf6925400 nid=0x223b runnable [junit4] [junit4] "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0xf6926800 nid=0x223f runnable [junit4] [junit4] "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0xf69bb800 nid=0x2249 runnable [junit4] [junit4] "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0xf69bcc00 nid=0x224b runnable [junit4] [junit4] "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0xf69be000 nid=0x224d runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0xf69b2c00 nid=0x2248 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0xf69b1800 nid=0x2247 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0xf69b0400 nid=0x2246 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0xf69af000 nid=0x2245 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0xf69ad800 nid=0x2244 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0xf69ac400 nid=0x2243 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0xf69ab000 nid=0x2242 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0xf69a9c00 nid=0x2241 runnable [junit4] [junit4] "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0xf69a8800 nid=0x2240 runnable [junit4] [junit4] "VM Periodic Task Thread" os_prio=0 tid=0xd0a2cc00 nid=0x2263 waiting on condition [junit4] [junit4] JNI global references: 2899 [junit4] [junit4] Heap [junit4] garbage-first heap total 524288K, used 109160K [0xd2a00000, 0xd2b00800, 0xf2a00000) [junit4] region size 1024K, 2 young (2048K), 1 survivors (1024K) [junit4] Metaspace used 50948K, capacity 52555K, committed 52608K, reserved 53552K [junit4] [junit4] <<< JVM J1: EOF ---- {noformat} I'd like to fix this as we now have a problem: - We'd like to start Lucene/SOlr 7.0 release - Java 9 final release candidate is there - Hadoop breaks in one of its most basic classes (Shell) - Updating Hadoop makes tests run, but hang in some deadlock while trying to interrupt them - A bugfix release fpr Hadopp 2.7.x might take long time [~markrmil...@gmail.com]: Do you have any idea what can cause above hangs on stack traces? I created a branch on my Github: https://github.com/apache/lucene-solr/compare/master...uschindler:jira/SOLR-10951 I also setup a Policeman Jenkins: https://jenkins.thetaphi.de/job/Lucene-Solr-Hadoop-Update/ (I triggered stack traces when it hug, but all look like the one above) > Update Hadoop dependencies to 2.8.1, so Solr works with Java 9 > -------------------------------------------------------------- > > Key: SOLR-10951 > URL: https://issues.apache.org/jira/browse/SOLR-10951 > Project: Solr > Issue Type: Task > Security Level: Public(Default Security Level. Issues are Public) > Components: Hadoop Integration, hdfs > Affects Versions: 6.6, master (7.0) > Reporter: Uwe Schindler > Assignee: Uwe Schindler > Priority: Blocker > Labels: Java9 > Fix For: master (7.0), 6.7 > > Attachments: SOLR-10951.patch > > > See issue: HADOOP-14586 > Since Java 9 build 175 (the first Java 9 relaese candidate), Hadoop > integration fails in Java 9: > {noformat} > [junit4] 2> 129956 ERROR (jetty-launcher-232-thread-2) [ ] > o.a.s.c.SolrCore null:java.lang.ExceptionInInitializerError > [junit4] 2> at > org.apache.hadoop.util.StringUtils.<clinit>(StringUtils.java:80) > [junit4] 2> at > org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1437) > [junit4] 2> at > org.apache.hadoop.security.token.delegation.web.DelegationTokenManager.<init>(DelegationTokenManager.java:115) > [junit4] 2> at > org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.initTokenManager(DelegationTokenAuthenticationHandler.java:148) > [junit4] 2> at > org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.init(DelegationTokenAuthenticationHandler.java:118) > [junit4] 2> at > org.apache.hadoop.security.authentication.server.AuthenticationFilter.initializeAuthHandler(AuthenticationFilter.java:238) > [junit4] 2> at > org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.initializeAuthHandler(DelegationTokenAuthenticationFilter.java:209) > [junit4] 2> at > org.apache.solr.security.HadoopAuthFilter.initializeAuthHandler(HadoopAuthFilter.java:120) > [junit4] 2> at > org.apache.hadoop.security.authentication.server.AuthenticationFilter.init(AuthenticationFilter.java:227) > [junit4] 2> at > org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.init(DelegationTokenAuthenticationFilter.java:175) > [junit4] 2> at > org.apache.solr.security.HadoopAuthFilter.init(HadoopAuthFilter.java:68) > [junit4] 2> at > org.apache.solr.security.HadoopAuthPlugin.init(HadoopAuthPlugin.java:142) > [junit4] 2> at > org.apache.solr.core.CoreContainer.initializeAuthenticationPlugin(CoreContainer.java:360) > [junit4] 2> at > org.apache.solr.core.CoreContainer.reloadSecurityProperties(CoreContainer.java:684) > [junit4] 2> at > org.apache.solr.core.CoreContainer.load(CoreContainer.java:522) > [junit4] 2> at > org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:257) > [junit4] 2> at > org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:177) > [junit4] 2> at > org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) > [junit4] 2> at > org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) > [junit4] 2> at > org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) > [junit4] 2> at > org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) > [junit4] 2> at > org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) > [junit4] 2> at > org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) > [junit4] 2> at > org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) > [junit4] 2> at > org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) > [junit4] 2> at > org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) > [junit4] 2> at > org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) > [junit4] 2> at > org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) > [junit4] 2> at > org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) > [junit4] 2> at > org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:382) > [junit4] 2> at > org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:245) > [junit4] 2> at > java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) > [junit4] 2> at > org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) > [junit4] 2> at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1161) > [junit4] 2> at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) > [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) > [junit4] 2> Caused by: java.lang.StringIndexOutOfBoundsException: begin > 0, end 3, length 1 > [junit4] 2> at > java.base/java.lang.String.checkBoundsBeginEnd(String.java:3116) > [junit4] 2> at > java.base/java.lang.String.substring(String.java:1885) > [junit4] 2> at org.apache.hadoop.util.Shell.<clinit>(Shell.java:51) > [junit4] 2> ... 36 more > [junit4] 2> > {noformat} > This means we can for now only disable all tests using Hadoop, if Java 9 is > detected. > On Policeman Jenkins the HDFS integration is already disabled, but there are > 2 more tests that fail: > - org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth > - org.apache.solr.cloud.TestSolrCloudWithDelegationTokens > I will add an assumeFalse for a while, until Hadoop fixed the issue. -- This message was sent by Atlassian JIRA (v6.4.14#64029) --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org