[
https://issues.apache.org/jira/browse/SOLR-10951?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16062375#comment-16062375
]
Uwe Schindler commented on SOLR-10951:
--------------------------------------
I had to change some additional dependencies, but I got it running up to the
place where tests of HDFS tear down.
With Hadoop-2.8.1, the test shutdown code and thread leak detector hangs
indefinite in a locking while trying to interrupt() threads:
{noformat}
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] 2017-06-25 15:58:44
[junit4] Full thread dump Java HotSpot(TM) Client VM (25.131-b11 mixed mode):
[junit4]
[junit4] "IPC Server handler 9 on 41067" #8132 daemon prio=5 os_prio=0
tid=0xc6674800 nid=0x67c3 waiting on condition [0xca932000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 8 on 41067" #8131 daemon prio=5 os_prio=0
tid=0xc6673c00 nid=0x67c2 waiting on condition [0xcf2ad000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 7 on 41067" #8130 daemon prio=5 os_prio=0
tid=0xc6672c00 nid=0x67c1 waiting on condition [0xcf25c000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awa
[junit4] itNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 6 on 41067" #8129 daemon prio=5 os_prio=0
tid=0xc6671c00 nid=0x67c0 waiting on condition [0xca515000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 5 on 41067" #8128 daemon prio=5 os_prio=0
tid=0xc6670c00 nid=0x67bf waiting on condition [0xcaa76000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 4 on 41067" #8127 daemon prio=5 os_prio=0
tid=0xc6670000 nid=0x67be waiting on condition [0xca5b7000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchro
[junit4] nizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 3 on 41067" #8126 daemon prio=5 os_prio=0
tid=0xc666f000 nid=0x67bd waiting on condition [0xca3d1000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 2 on 41067" #8125 daemon prio=5 os_prio=0
tid=0xc666e400 nid=0x67bc waiting on condition [0xce4ad000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 1 on 41067" #8124 daemon prio=5 os_prio=0
tid=0xc666d400 nid=0x67bb waiting on condition [0xca473000]
[junit4] j
[junit4] ava.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server handler 0 on 41067" #8123 daemon prio=5 os_prio=0
tid=0xc666c400 nid=0x67ba waiting on condition [0xc9ffe000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e000> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:214)
[junit4] at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2441)
[junit4]
[junit4] "IPC Server listener on 41067" #8114 daemon prio=5 os_prio=0
tid=0xc666bc00 nid=0x67b9 runnable [0xc7018000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4] at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] at
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] - locked <0xd7ce8eb0> (a java.lang.Object)
[junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] - locked <0xd7ce8c70> (a sun.nio.ch.Util$3)
[junit4] - locked <0xd7ce8c60> (a java.util.Collections$UnmodifiableSet)
[junit4] - locked <0xd7ce8b28> (a sun.nio.ch.EPollSelectorImpl)
[junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] at org.apache.hadoop.ipc.Server$Listener.r
[junit4] un(Server.java:1036)
[junit4]
[junit4] "IPC Server Responder" #8117 daemon prio=5 os_prio=0 tid=0xc666b400
nid=0x67b8 runnable [0xc8f8a000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] at
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] - locked <0xd7b3e3e8> (a sun.nio.ch.Util$3)
[junit4] - locked <0xd7b3e3f8> (a java.util.Collections$UnmodifiableSet)
[junit4] - locked <0xd7b3e398> (a sun.nio.ch.EPollSelectorImpl)
[junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] at
org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1214)
[junit4] at org.apache.hadoop.ipc.Server$Responder.run(Server.java:1197)
[junit4]
[junit4] "Block report processor" #8112 daemon prio=5 os_prio=0
tid=0xc6662800 nid=0x67b7 waiting on condition [0xca1eb000]
[junit4] java.lang.Thread.State: WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e590> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] at
java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
[junit4] at
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:3984)
[junit4] at
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:3973)
[junit4]
[junit4]
"org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@1e1bd5d"
#8113 daemon prio=5 os_prio=0 tid=0xc665ec00 nid=0x67b5 waiting on condition
[0xc70ba000]
[junit4] java.lang.Thread.State: TIMED_WAITING (sleeping)
[junit4] at java.lang.Thread.sleep(Native Method)
[junit4] at
org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:419)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4]
[junit4] "DecommissionMonitor-0" #8122 d
[junit4] aemon prio=5 os_prio=0 tid=0xc665e000 nid=0x67b4 waiting on
condition [0xcf1ba000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3e9b8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4]
[junit4]
"org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@1964584"
#8121 daemon prio=5 os_prio=0 tid=0xc665c400 nid=0x67b3 waiting on condition
[0xc8df5000]
[junit4] java.lang.Thread.State: TIMED_WAITING (sleeping)
[junit4] at java.lang.Thread.sleep(Native Method)
[junit4] at
org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:237)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4]
[junit4] "IPC Server idle connection scanner for port 41067" #8116 daemon
prio=5 os_prio=0 tid=0xc661b400 nid=0x67ad in Object.wait() [0xcb2fe000]
[junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.util.TimerThread.mainLoop(Timer.java:552)
[junit4] - locked <0xd7b3ed88> (a java.util.TaskQueue)
[junit4] at java.util.TimerThread.run(Timer.java:505)
[junit4]
[junit4] "Socket Reader #1 for port 41067" #8115 prio=5 os_prio=0
tid=0xc6619400 nid=0x67ac runnable [0xc9c32000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4] at sun.nio.ch.EPollArrayWrapper.interrupt(Na
[junit4] tive Method)
[junit4] at
sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] at
sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] - locked <0xd7b3ef60> (a java.lang.Object)
[junit4] at
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] at
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] at
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] - locked <0xd7b3ef68> (a sun.nio.ch.Util$3)
[junit4] - locked <0xd7b3ef78> (a java.util.Collections$UnmodifiableSet)
[junit4] - locked <0xd7b3ef20> (a sun.nio.ch.EPollSelectorImpl)
[junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:974)
[junit4] - locked <0xd7b3ed98> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
[junit4] at
org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:953)
[junit4]
[junit4] "Timer-1" #8110 daemon prio=5 os_prio=0 tid=0xc7626400 nid=0x67ab
in Object.wait() [0xcb069000]
[junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.util.TimerThread.mainLoop(Timer.java:552)
[junit4] - locked <0xd7b3f108> (a java.util.TaskQueue)
[junit4] at java.util.TimerThread.run(Timer.java:505)
[junit4]
[junit4] "Timer-0" #8109 daemon prio=5 os_prio=0 tid=0xc760e000 nid=0x67aa
in Object.wait() [0xc8f39000]
[junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.util.TimerThread.mainLoop(Timer.java:552)
[junit4] - locked <0xd7b3f2c8> (a java.util.TaskQueue)
[junit4] at java.util.TimerThread.run(Timer.java:505)
[junit4]
[junit4] "13142186@qtp-24418277-1 - Acceptor0
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost.localdomain:43319"
#8107 daemon prio=5 os_prio=0 tid=0xc7603800 nid=0x67a8 runnable [0xcaac7000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] at sun.nio.ch.EPol
[junit4] lSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] - locked <0xd7e0b980> (a sun.nio.ch.Util$3)
[junit4] - locked <0xd7e0b990> (a java.util.Collections$UnmodifiableSet)
[junit4] - locked <0xd7e0b940> (a sun.nio.ch.EPollSelectorImpl)
[junit4] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
[junit4] at
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
[junit4] at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
[junit4] at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
[junit4] at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
[junit4]
[junit4] "pool-2-thread-1" #8105 prio=5 os_prio=0 tid=0xc7a73400 nid=0x67a6
waiting on condition [0xce1c7000]
[junit4] java.lang.Thread.State: TIMED_WAITING (parking)
[junit4] at sun.misc.Unsafe.park(Native Method)
[junit4] - parking to wait for <0xd7b3f7f8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
[junit4] at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4]
[junit4] "Timer for 'NameNode' metrics system" #8102 daemon prio=5 os_prio=0
tid=0xc7a52000 nid=0x67a4 in Object.wait() [0xc9b3f000]
[junit4] java.lang.Thread.State: TIMED_WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.util.TimerThread.mainLoop(Timer.java:552)
[junit4] - locked <0xd7b3fa38> (a java.util.TaskQueue)
[junit4] at java.util.TimerThread.run(Timer.java:505)
[junit4]
[junit4] "SUITE-HdfsDirectoryFactoryTest-seed#[61471115528599BC]" #8093
prio=10 os_prio=0 tid=0xcaec4400 nid=0x679d waiting for monitor entry
[0xcc159000]
[junit4] java.lang.Thread.State: BLOCKED (on object monitor)
[junit4] at java.lang.Thread.join(Thread.java:1243)
[junit4] - waiting to lock <0xd7b3ed98> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
[junit4] at
com.carrotsearch.randomizedtesting.ThreadLeakControl.tryToInterruptAll(ThreadLeakControl.java:776)
[junit4] at
com.carrotsearch.randomizedtesting.ThreadLeakControl.checkThreadLeaks(ThreadLeakControl.java:626)
[junit4] at
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:439)
[junit4] at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] at
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] at
com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4]
[junit4] "TimeLimitedCollector timer thread" #2531 daemon prio=5 os_prio=0
tid=0xf2cd9800 nid=0x3d98 waiting on condition [0xc96d1000]
[junit4] java.lang.Thread.State: TIMED_WAITING (sleeping)
[junit4] at java.lang.Thread.sleep(Native Method)
[junit4] at
org.apache.lucene.search.TimeLimitingCollector$TimerThread.run(TimeLimitingCollector.java:261)
[junit4]
[junit4] "JUnit4-serializer-daemon" #8 daemon prio=5 os_prio=0
tid=0xd027cc00 nid=0x2266 waiting on condition [0xd147b000]
[junit4] java.lang.Thread.State: TIMED_WAITING (sleeping)
[junit4] at java.lang.Thread.sleep(Native Method)
[junit4] at
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4]
[junit4] "Service Thread" #7 daemon prio=9 os_prio=0 tid=0xd0a2a400
nid=0x2261 runnable [0x00000000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4]
[junit4] "C1 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0xd0a26c00
nid=0x225f waiting on condition [0x00000000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4]
[junit4] "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0xd0a25400
nid=0x225e waiting on condition [0x00000000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4]
[junit4] "Surrogate Locker Thre
[junit4] ad (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0xd0a23c00
nid=0x225d waiting on condition [0x00000000]
[junit4] java.lang.Thread.State: RUNNABLE
[junit4]
[junit4] "Finalizer" #3 daemon prio=8 os_prio=0 tid=0xd13d9400 nid=0x2254 in
Object.wait() [0xf3b7d000]
[junit4] java.lang.Thread.State: WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
[junit4] - locked <0xd301c770> (a java.lang.ref.ReferenceQueue$Lock)
[junit4] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
[junit4] at
java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
[junit4]
[junit4] "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0xd13d6800
nid=0x2253 in Object.wait() [0xf3d7d000]
[junit4] java.lang.Thread.State: WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.lang.Object.wait(Object.java:502)
[junit4] at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
[junit4] - locked <0xd301c910> (a java.lang.ref.Reference$Lock)
[junit4] at
java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
[junit4]
[junit4] "main" #1 prio=5 os_prio=0 tid=0xf6912c00 nid=0x2214 in
Object.wait() [0xf6a65000]
[junit4] java.lang.Thread.State: WAITING (on object monitor)
[junit4] at java.lang.Object.wait(Native Method)
[junit4] at java.lang.Thread.join(Thread.java:1252)
[junit4] - locked <0xd7b3fa78> (a
com.carrotsearch.randomizedtesting.RandomizedRunner$2)
[junit4] at java.lang.Thread.join(Thread.java:1326)
[junit4] at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:244)
[junit4] at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
[junit4] at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4]
[junit4] "VM Thread" os_prio=0 tid=0xd13d1c00 nid=0x2252 runnable
[junit4]
[junit4] "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0xf691d800
nid=0x2229 runnable
[junit4]
[junit4] "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0xf691ec00
nid=0x222c runnable
[junit4]
[junit4] "Gang worker#2
[junit4] (Parallel GC Threads)" os_prio=0 tid=0xf6920000 nid=0x222f runnable
[junit4]
[junit4] "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0xf6921400
nid=0x2232 runnable
[junit4]
[junit4] "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0xf6922c00
nid=0x2235 runnable
[junit4]
[junit4] "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0xf6924000
nid=0x2238 runnable
[junit4]
[junit4] "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0xf6925400
nid=0x223b runnable
[junit4]
[junit4] "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0xf6926800
nid=0x223f runnable
[junit4]
[junit4] "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0xf69bb800
nid=0x2249 runnable
[junit4]
[junit4] "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0
tid=0xf69bcc00 nid=0x224b runnable
[junit4]
[junit4] "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0
tid=0xf69be000 nid=0x224d runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0xf69b2c00
nid=0x2248 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0xf69b1800
nid=0x2247 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0xf69b0400
nid=0x2246 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0xf69af000
nid=0x2245 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0xf69ad800
nid=0x2244 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0xf69ac400
nid=0x2243 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0xf69ab000
nid=0x2242 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0xf69a9c00
nid=0x2241 runnable
[junit4]
[junit4] "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0xf69a8800
nid=0x2240 runnable
[junit4]
[junit4] "VM Periodic Task Thread" os_prio=0 tid=0xd0a2cc00 nid=0x2263
waiting on condition
[junit4]
[junit4] JNI global references: 2899
[junit4]
[junit4] Heap
[junit4] garbage-first heap total 524288K, used 109160K [0xd2a00000,
0xd2b00800, 0xf2a00000)
[junit4] region size 1024K, 2 young (2048K), 1 survivors (1024K)
[junit4] Metaspace used 50948K, capacity 52555K, committed 52608K,
reserved 53552K
[junit4]
[junit4] <<< JVM J1: EOF ----
{noformat}
I'd like to fix this as we now have a problem:
- We'd like to start Lucene/SOlr 7.0 release
- Java 9 final release candidate is there
- Hadoop breaks in one of its most basic classes (Shell)
- Updating Hadoop makes tests run, but hang in some deadlock while trying to
interrupt them
- A bugfix release fpr Hadopp 2.7.x might take long time
[[email protected]]: Do you have any idea what can cause above hangs on
stack traces?
I created a branch on my Github:
https://github.com/apache/lucene-solr/compare/master...uschindler:jira/SOLR-10951
I also setup a Policeman Jenkins:
https://jenkins.thetaphi.de/job/Lucene-Solr-Hadoop-Update/
(I triggered stack traces when it hug, but all look like the one above)
> Update Hadoop dependencies to 2.8.1, so Solr works with Java 9
> --------------------------------------------------------------
>
> Key: SOLR-10951
> URL: https://issues.apache.org/jira/browse/SOLR-10951
> Project: Solr
> Issue Type: Task
> Security Level: Public(Default Security Level. Issues are Public)
> Components: Hadoop Integration, hdfs
> Affects Versions: 6.6, master (7.0)
> Reporter: Uwe Schindler
> Assignee: Uwe Schindler
> Priority: Blocker
> Labels: Java9
> Fix For: master (7.0), 6.7
>
> Attachments: SOLR-10951.patch
>
>
> See issue: HADOOP-14586
> Since Java 9 build 175 (the first Java 9 relaese candidate), Hadoop
> integration fails in Java 9:
> {noformat}
> [junit4] 2> 129956 ERROR (jetty-launcher-232-thread-2) [ ]
> o.a.s.c.SolrCore null:java.lang.ExceptionInInitializerError
> [junit4] 2> at
> org.apache.hadoop.util.StringUtils.<clinit>(StringUtils.java:80)
> [junit4] 2> at
> org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1437)
> [junit4] 2> at
> org.apache.hadoop.security.token.delegation.web.DelegationTokenManager.<init>(DelegationTokenManager.java:115)
> [junit4] 2> at
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.initTokenManager(DelegationTokenAuthenticationHandler.java:148)
> [junit4] 2> at
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.init(DelegationTokenAuthenticationHandler.java:118)
> [junit4] 2> at
> org.apache.hadoop.security.authentication.server.AuthenticationFilter.initializeAuthHandler(AuthenticationFilter.java:238)
> [junit4] 2> at
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.initializeAuthHandler(DelegationTokenAuthenticationFilter.java:209)
> [junit4] 2> at
> org.apache.solr.security.HadoopAuthFilter.initializeAuthHandler(HadoopAuthFilter.java:120)
> [junit4] 2> at
> org.apache.hadoop.security.authentication.server.AuthenticationFilter.init(AuthenticationFilter.java:227)
> [junit4] 2> at
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.init(DelegationTokenAuthenticationFilter.java:175)
> [junit4] 2> at
> org.apache.solr.security.HadoopAuthFilter.init(HadoopAuthFilter.java:68)
> [junit4] 2> at
> org.apache.solr.security.HadoopAuthPlugin.init(HadoopAuthPlugin.java:142)
> [junit4] 2> at
> org.apache.solr.core.CoreContainer.initializeAuthenticationPlugin(CoreContainer.java:360)
> [junit4] 2> at
> org.apache.solr.core.CoreContainer.reloadSecurityProperties(CoreContainer.java:684)
> [junit4] 2> at
> org.apache.solr.core.CoreContainer.load(CoreContainer.java:522)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:257)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:177)
> [junit4] 2> at
> org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
> [junit4] 2> at
> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
> [junit4] 2> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
> [junit4] 2> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
> [junit4] 2> at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
> [junit4] 2> at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
> [junit4] 2> at
> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:382)
> [junit4] 2> at
> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:245)
> [junit4] 2> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] 2> at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
> [junit4] 2> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1161)
> [junit4] 2> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
> [junit4] 2> Caused by: java.lang.StringIndexOutOfBoundsException: begin
> 0, end 3, length 1
> [junit4] 2> at
> java.base/java.lang.String.checkBoundsBeginEnd(String.java:3116)
> [junit4] 2> at
> java.base/java.lang.String.substring(String.java:1885)
> [junit4] 2> at org.apache.hadoop.util.Shell.<clinit>(Shell.java:51)
> [junit4] 2> ... 36 more
> [junit4] 2>
> {noformat}
> This means we can for now only disable all tests using Hadoop, if Java 9 is
> detected.
> On Policeman Jenkins the HDFS integration is already disabled, but there are
> 2 more tests that fail:
> - org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth
> - org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
> I will add an assumeFalse for a while, until Hadoop fixed the issue.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
