[JENKINS] Lucene » Lucene-Solr-Tests-8.11 - Build # 335 - Unstable!

[Apache Jenkins Server]

Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-Tests-8.11/335/

12 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
Too many failed volumes - current valid volumes: 0, volumes configured: 2, 
volumes failed: 2, volume failures tolerated: 0

Stack Trace:
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
        at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
        at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest.setupClass(HdfsNNFailoverTest.java:45)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.lang.Thread.run(Thread.java:750)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
2 threads leaked from SUITE scope at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest:     1) Thread[id=2898, 
name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=2899, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=2899, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=2898, name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=2899, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=2899, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest

Error Message:
Too many failed volumes - current valid volumes: 0, volumes configured: 2, 
volumes failed: 2, volume failures tolerated: 0

Stack Trace:
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
        at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:87)
        at 
org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest.beforeClass(HdfsRecoverLeaseTest.java:54)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.lang.Thread.run(Thread.java:750)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest

Error Message:
2 threads leaked from SUITE scope at 
org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest:     1) Thread[id=1584, 
name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-HdfsRecoverLeaseTest]         at sun.misc.Unsafe.park(Native Method) 
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=1585, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-HdfsRecoverLeaseTest]         at sun.misc.Unsafe.park(Native Method) 
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest: 
   1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=1584, name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-HdfsRecoverLeaseTest]         at sun.misc.Unsafe.park(Native Method) 
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=1585, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-HdfsRecoverLeaseTest]         at sun.misc.Unsafe.park(Native Method) 
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.index.hdfs.CheckHdfsIndexTest

Error Message:
Too many failed volumes - current valid volumes: 0, volumes configured: 2, 
volumes failed: 2, volume failures tolerated: 0

Stack Trace:
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
        at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:83)
        at 
org.apache.solr.index.hdfs.CheckHdfsIndexTest.setupClass(CheckHdfsIndexTest.java:70)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.lang.Thread.run(Thread.java:750)


FAILED:  junit.framework.TestSuite.org.apache.solr.index.hdfs.CheckHdfsIndexTest

Error Message:
2 threads leaked from SUITE scope at 
org.apache.solr.index.hdfs.CheckHdfsIndexTest:     1) Thread[id=11013, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-CheckHdfsIndexTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=11012, 
name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-CheckHdfsIndexTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.index.hdfs.CheckHdfsIndexTest: 
   1) Thread[id=11013, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=11012, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.index.hdfs.CheckHdfsIndexTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=11013, name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-CheckHdfsIndexTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=11012, 
name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-CheckHdfsIndexTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=11013, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=11012, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestRecoveryHdfs

Error Message:
Too many failed volumes - current valid volumes: 0, volumes configured: 2, 
volumes failed: 2, volume failures tolerated: 0

Stack Trace:
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
        at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:83)
        at 
org.apache.solr.search.TestRecoveryHdfs.beforeClass(TestRecoveryHdfs.java:89)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.lang.Thread.run(Thread.java:750)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestRecoveryHdfs

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.search.TestRecoveryHdfs:   
  1) Thread[id=8562, name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-TestRecoveryHdfs]         at sun.misc.Unsafe.park(Native Method)     
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)   
      at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=8563, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-TestRecoveryHdfs]         at sun.misc.Unsafe.park(Native Method)     
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)   
      at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.search.TestRecoveryHdfs: 
   1) Thread[id=8562, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-TestRecoveryHdfs]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=8563, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-TestRecoveryHdfs]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestRecoveryHdfs

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=8562, name=StorageLocationChecker thread 0, state=TIMED_WAITING, 
group=TGRP-TestRecoveryHdfs]         at sun.misc.Unsafe.park(Native Method)     
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)   
      at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)    2) Thread[id=8563, 
name=StorageLocationChecker thread 1, state=TIMED_WAITING, 
group=TGRP-TestRecoveryHdfs]         at sun.misc.Unsafe.park(Native Method)     
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)   
      at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:750)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=8562, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-TestRecoveryHdfs]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
   2) Thread[id=8563, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-TestRecoveryHdfs]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:750)
        at __randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)




Build Log:
[...truncated 13626 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest
   [junit4]   2> 97500 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to 
test-framework derived value of 
'/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 97500 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 97501 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.u.ErrorLogMuter Closing ErrorLogMuter-regex-20 after mutting 0 log 
messages
   [junit4]   2> 97501 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-21 for ERROR logs matching 
regex: ignore_exception
   [junit4]   2> 97502 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Created dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/data-dir-9-001
   [junit4]   2> 97502 WARN  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=25 numCloses=25
   [junit4]   2> 97502 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 97504 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 99394 WARN  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.h.s.a.s.AuthenticationFilter Unable to initialize FileSignerSecretProvider, 
falling back to use random secrets. Reason: access denied 
("java.io.FilePermission" "/home/jenkins/hadoop-http-auth-signature-secret" 
"read")
   [junit4]   2> 99405 WARN  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 99440 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.Server jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 
27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 1.8.0_362-b09
   [junit4]   2> 99447 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 99447 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 99447 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 99450 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@74116ba5{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 99663 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@518e5a6{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/jetty-localhost_localdomain-44209-hadoop-hdfs-3_2_4-tests_jar-_-any-5144893170774452128/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/hdfs}
   [junit4]   2> 99664 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.AbstractConnector Started ServerConnector@6cb0e9eb{HTTP/1.1, 
(http/1.1)}{localhost.localdomain:44209}
   [junit4]   2> 99664 INFO  
(SUITE-HdfsRecoverLeaseTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.Server Started @99699ms
   [junit4]   2> 100585 WARN  (StorageLocationChecker thread 0) [     ] 
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   2> 100603 WARN  (Listener at localhost.localdomain/45539) [     ] 
o.a.h.h.s.d.c.StorageLocationChecker Exception checking StorageLocation 
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data1
   [junit4]   2>           => 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Directory is not 
executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data1
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
   [junit4]   2> org.apache.hadoop.util.DiskChecker$DiskErrorException: 
Directory is not executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data1
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
 ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDirInternal(DiskChecker.java:144) 
~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDir(DiskChecker.java:118) ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:239)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:52)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.checker.ThrottledAsyncChecker$1.call(ThrottledAsyncChecker.java:142)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:76)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
~[?:1.8.0_362]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
~[?:1.8.0_362]
   [junit4]   2>        at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
   [junit4]   2> 100606 WARN  (Listener at localhost.localdomain/45539) [     ] 
o.a.h.h.s.d.c.StorageLocationChecker Exception checking StorageLocation 
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data2
   [junit4]   2>           => 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Directory is not 
executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data2
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
   [junit4]   2> org.apache.hadoop.util.DiskChecker$DiskErrorException: 
Directory is not executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoverLeaseTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data2
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
 ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDirInternal(DiskChecker.java:144) 
~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDir(DiskChecker.java:118) ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:239)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:52)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.checker.ThrottledAsyncChecker$1.call(ThrottledAsyncChecker.java:142)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:76)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
~[?:1.8.0_362]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
~[?:1.8.0_362]
   [junit4]   2>        at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
   [junit4]   2> 100618 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.w.WebAppContext@518e5a6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/hdfs}
   [junit4]   2> 100619 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.e.j.s.AbstractConnector Stopped ServerConnector@6cb0e9eb{HTTP/1.1, 
(http/1.1)}{localhost.localdomain:0}
   [junit4]   2> 100619 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 100619 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@74116ba5{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/static,STOPPED}
   [junit4]   2> 100628 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.a.s.u.ErrorLogMuter Closing ErrorLogMuter-regex-21 after mutting 0 log 
messages
   [junit4]   2> 100628 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-22 for ERROR logs matching 
regex: ignore_exception
   [junit4]   2> 100628 INFO  (Listener at localhost.localdomain/45539) [     ] 
o.a.s.SolrTestCaseJ4 ------------------------------------------------------- 
Done waiting for tracked resources to be released
   [junit4]   2> Oct 30, 2023 8:14:12 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked 
thread(s).
   [junit4]   2> Oct 30, 2023 8:14:22 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at 
org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest: 
   [junit4]   2>    1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2>    2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2> Oct 30, 2023 8:14:22 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]   2>    2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]   2> Oct 30, 2023 8:14:24 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be 
terminated:
   [junit4]   2>    1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2>    2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene87): {}, 
docValues:{}, maxPointsInLeafNode=2015, maxMBSortInHeap=7.394502662065944, 
sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=es-NI, 
timezone=Europe/Samara
   [junit4]   2> NOTE: Linux 4.15.0-213-generic amd64/Temurin 1.8.0_362 
(64-bit)/cpus=4,threads=4,free=294577144,total=454033408
   [junit4]   2> NOTE: All tests run in this JVM: 
[JavaBinAtomicUpdateMultivalueTest, PrimitiveFieldTypeTest, 
UpdateRequestProcessorFactoryTest, TestBlobHandler, SearchRateTriggerTest, 
TestDocBasedVersionConstraints, ZkDynamicConfigTest, TestCloudRecovery, 
TestFieldCacheSort, HdfsRecoverLeaseTest]
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=HdfsRecoverLeaseTest -Dtests.seed=E12EF9EFE4D689E4 
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-NI 
-Dtests.timezone=Europe/Samara -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | HdfsRecoverLeaseTest (suite) <<<
   [junit4]    > Throwable #1: 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:87)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest.beforeClass(HdfsRecoverLeaseTest.java:54)
   [junit4]    >        at java.lang.Thread.run(Thread.java:750)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.hdfs.HdfsRecoverLeaseTest: 
   [junit4]    >    1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >    2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)Throwable #3: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=1584, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >    2) Thread[id=1585, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsRecoverLeaseTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
   [junit4] Completed [32/958 (1!)] on J2 in 15.73s, 0 tests, 3 errors <<< 
FAILURES!

[...truncated 982 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsNNFailoverTest
   [junit4]   2> 146929 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to 
test-framework derived value of 
'/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 146929 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 146929 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.u.ErrorLogMuter Closing ErrorLogMuter-regex-31 after mutting 0 log 
messages
   [junit4]   2> 146929 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-32 for ERROR logs matching 
regex: ignore_exception
   [junit4]   2> 146930 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Created dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/data-dir-13-001
   [junit4]   2> 146930 WARN  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 146930 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 146932 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 146932 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 148723 WARN  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.h.s.a.s.AuthenticationFilter Unable to initialize FileSignerSecretProvider, 
falling back to use random secrets. Reason: access denied 
("java.io.FilePermission" "/home/jenkins/hadoop-http-auth-signature-secret" 
"read")
   [junit4]   2> 148735 WARN  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 148769 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.Server jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 
27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 1.8.0_362-b09
   [junit4]   2> 148772 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 148772 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 148772 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 148775 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@17a6aa4e{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 148987 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@dfabb6{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/jetty-localhost_localdomain-43743-hadoop-hdfs-3_2_4-tests_jar-_-any-3698874178786877315/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/hdfs}
   [junit4]   2> 148988 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.AbstractConnector Started ServerConnector@7b63ab03{HTTP/1.1, 
(http/1.1)}{localhost.localdomain:43743}
   [junit4]   2> 148988 INFO  
(SUITE-HdfsNNFailoverTest-seed#[E12EF9EFE4D689E4]-worker) [     ] 
o.e.j.s.Server Started @149042ms
   [junit4]   2> 149983 WARN  (StorageLocationChecker thread 0) [     ] 
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   2> 149992 WARN  (Listener at localhost.localdomain/42427) [     ] 
o.a.h.h.s.d.c.StorageLocationChecker Exception checking StorageLocation 
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data1
   [junit4]   2>           => 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Directory is not 
executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data1
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
   [junit4]   2> org.apache.hadoop.util.DiskChecker$DiskErrorException: 
Directory is not executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data1
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
 ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDirInternal(DiskChecker.java:144) 
~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDir(DiskChecker.java:118) ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:239)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:52)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.checker.ThrottledAsyncChecker$1.call(ThrottledAsyncChecker.java:142)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:76)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
~[?:1.8.0_362]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
~[?:1.8.0_362]
   [junit4]   2>        at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
   [junit4]   2> 149993 WARN  (Listener at localhost.localdomain/42427) [     ] 
o.a.h.h.s.d.c.StorageLocationChecker Exception checking StorageLocation 
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data2
   [junit4]   2>           => 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Directory is not 
executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data2
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
   [junit4]   2> org.apache.hadoop.util.DiskChecker$DiskErrorException: 
Directory is not executable: 
/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.11/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsNNFailoverTest_E12EF9EFE4D689E4-001/tempDir-001/hdfsBaseDir/data/data2
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkAccessByFileMethods(DiskChecker.java:174)
 ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDirInternal(DiskChecker.java:144) 
~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.util.DiskChecker.checkDir(DiskChecker.java:118) ~[test/:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:239)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.StorageLocation.check(StorageLocation.java:52)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.datanode.checker.ThrottledAsyncChecker$1.call(ThrottledAsyncChecker.java:142)
 ~[hadoop-hdfs-3.2.4.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:76)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
 ~[guava-32.1.3-jre.jar:?]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
~[?:1.8.0_362]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
~[?:1.8.0_362]
   [junit4]   2>        at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
   [junit4]   2> 150009 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.w.WebAppContext@dfabb6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/hdfs}
   [junit4]   2> 150009 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.e.j.s.AbstractConnector Stopped ServerConnector@7b63ab03{HTTP/1.1, 
(http/1.1)}{localhost.localdomain:0}
   [junit4]   2> 150009 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 150010 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@17a6aa4e{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.4-tests.jar!/webapps/static,STOPPED}
   [junit4]   2> 150018 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.a.s.u.ErrorLogMuter Closing ErrorLogMuter-regex-32 after mutting 0 log 
messages
   [junit4]   2> 150018 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-33 for ERROR logs matching 
regex: ignore_exception
   [junit4]   2> 150018 INFO  (Listener at localhost.localdomain/42427) [     ] 
o.a.s.SolrTestCaseJ4 ------------------------------------------------------- 
Done waiting for tracked resources to be released
   [junit4]   2> Oct 30, 2023 8:15:01 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked 
thread(s).
   [junit4]   2> Oct 30, 2023 8:15:11 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   [junit4]   2>    1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2>    2) Thread[id=2899, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2> Oct 30, 2023 8:15:11 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    2) Thread[id=2899, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2> Oct 30, 2023 8:15:14 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be 
terminated:
   [junit4]   2>    1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2>    2) Thread[id=2899, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2> NOTE: test params are: codec=Lucene87, 
sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=et, 
timezone=America/Manaus
   [junit4]   2> NOTE: Linux 4.15.0-213-generic amd64/Temurin 1.8.0_362 
(64-bit)/cpus=4,threads=4,free=155186544,total=461373440
   [junit4]   2> NOTE: All tests run in this JVM: [RequestHandlersTest, 
TestCloudSchemaless, TriggerCooldownIntegrationTest, CoreAdminHandlerTest, 
TimeRoutedAliasUpdateProcessorTest, IndexSizeTriggerTest, 
ClassificationUpdateProcessorTest, TestHashPartitioner, 
TestShardHandlerFactory, TestTrackingShardHandlerFactory, 
UnloadDistributedZkTest, MetricsDisabledCloudTest, ShardRoutingTest, 
TestManagedResourceStorage, HdfsNNFailoverTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsNNFailoverTest 
-Dtests.seed=E12EF9EFE4D689E4 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=et -Dtests.timezone=America/Manaus -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | HdfsNNFailoverTest (suite) <<<
   [junit4]    > Throwable #1: 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest.setupClass(HdfsNNFailoverTest.java:45)
   [junit4]    >        at java.lang.Thread.run(Thread.java:750)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   [junit4]    >    1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >    2) Thread[id=2899, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)Throwable #3: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=2898, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$Trans

[...truncated too long message...]

geLocationChecker thread 0, state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]   2> Oct 30, 2023 8:22:23 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be 
terminated:
   [junit4]   2>    1) Thread[id=11013, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2>    2) Thread[id=11012, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:750)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene87): {}, 
docValues:{}, maxPointsInLeafNode=1253, maxMBSortInHeap=7.831291849231974, 
sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=ar-EG, timezone=ART
   [junit4]   2> NOTE: Linux 4.15.0-213-generic amd64/Temurin 1.8.0_362 
(64-bit)/cpus=4,threads=4,free=98345648,total=503316480
   [junit4]   2> NOTE: All tests run in this JVM: [NestedAtomicUpdateTest, 
RankFieldTest, XMLAtomicUpdateMultivalueTest, TestCSVLoader, 
SystemLogListenerTest, TestNamedUpdateProcessors, AlternateDirectoryTest, 
TestCloudRecovery2, TestFieldCacheSortRandom, HdfsRecoveryZkTest, 
ComputePlanActionTest, TestComplexPhraseLeadingWildcard, 
CollectionsAPIDistributedZkTest, SplitHandlerTest, ZkShardTermsTest, 
PrimUtilsTest, SolrXmlInZkTest, EnumFieldTest, DateRangeFieldTest, 
TestConfigSetsAPIExclusivity, HdfsChaosMonkeyNothingIsSafeTest, 
SolrJmxReporterCloudTest, TestDistributedGrouping, 
TestDelegationWithHadoopAuth, TestCoreContainer, TestLegacyFieldReuse, 
LeaderFailoverAfterPartitionTest, TestSchemaNameResource, 
TestAuthorizationFramework, TestMacros, TestExportWriter, 
TestNonDefinedSimilarityFactory, TestRandomCollapseQParserPlugin, 
SaslZkACLProviderTest, HighlighterTest, CursorPagingTest, 
TriggerIntegrationTest, URLClassifyProcessorTest, 
TestCollectionsAPIViaSolrCloudCluster, DateMathParserTest, 
DocumentAnalysisRequestHandlerTest, CheckHdfsIndexTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CheckHdfsIndexTest 
-Dtests.seed=E12EF9EFE4D689E4 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=ar-EG -Dtests.timezone=ART -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J3 | CheckHdfsIndexTest (suite) <<<
   [junit4]    > Throwable #1: 
org.apache.hadoop.util.DiskChecker$DiskErrorException: Too many failed volumes 
- current valid volumes: 0, volumes configured: 2, volumes failed: 2, volume 
failures tolerated: 0
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker.check(StorageLocationChecker.java:231)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2813)
   [junit4]    >        at 
org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2728)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1740)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:954)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:561)
   [junit4]    >        at 
org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:503)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:198)
   [junit4]    >        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:83)
   [junit4]    >        at 
org.apache.solr.index.hdfs.CheckHdfsIndexTest.setupClass(CheckHdfsIndexTest.java:70)
   [junit4]    >        at java.lang.Thread.run(Thread.java:750)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.index.hdfs.CheckHdfsIndexTest: 
   [junit4]    >    1) Thread[id=11013, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >    2) Thread[id=11012, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)Throwable #3: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=11013, name=StorageLocationChecker thread 1, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >    2) Thread[id=11012, name=StorageLocationChecker thread 0, 
state=TIMED_WAITING, group=TGRP-CheckHdfsIndexTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:750)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E12EF9EFE4D689E4]:0)
   [junit4] Completed [881/958 (4!)] on J3 in 15.25s, 0 tests, 3 errors <<< 
FAILURES!

[...truncated 54624 lines...]

---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscr...@lucene.apache.org
For additional commands, e-mail: builds-h...@lucene.apache.org