Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1065/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew

Error Message:
expected:<200> but was:<403>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<403>
        at 
__randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:8AB134D6220492B5]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.renewDelegationToken(TestSolrCloudWithDelegationTokens.java:130)
        at 
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.verifyDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:315)
        at 
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:332)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsNNFailoverTest.test

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 
127.0.0.1:63593 within 45000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: 
Could not connect to ZooKeeper 127.0.0.1:63593 within 45000 ms
        at 
__randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:357EFF12B43422E9]:0)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
        at 
org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:86)
        at 
org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:80)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.distribSetUp(AbstractDistribZkTestBase.java:81)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribSetUp(AbstractFullDistribZkTestBase.java:219)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:955)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to 
ZooKeeper 127.0.0.1:63593 within 45000 ms
        at 
org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
        ... 38 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
5 threads leaked from SUITE scope at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest:     1) Thread[id=1610, 
name=SyncThread:0, state=WAITING, group=TGRP-HdfsNNFailoverTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
    2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, 
state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]         at 
sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at 
sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at 
sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1607, 
name=Thread-359, state=WAITING, group=TGRP-HdfsNNFailoverTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Thread.join(Thread.java:1249)         at 
java.lang.Thread.join(Thread.java:1323)         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
         at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    
4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)      
   at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
    5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, 
group=TGRP-HdfsNNFailoverTest]         at java.lang.Object.wait(Native Method)  
       at 
org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE 
scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   1) Thread[id=1610, name=SyncThread:0, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, 
state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1607, name=Thread-359, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1249)
        at java.lang.Thread.join(Thread.java:1323)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, 
group=TGRP-HdfsNNFailoverTest]
        at java.lang.Object.wait(Native Method)
        at 
org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
        at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-HdfsNNFailoverTest]         at 
sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)         at 
sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)         
at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)         
at 
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)   
      at 
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)         
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, 
state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
        at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
        at 
sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
        at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
        at 
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at 
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:745)
        at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)




Build Log:
[...truncated 10826 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/init-core-data-001
   [junit4]   2> 172675 INFO  
(SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 172675 INFO  
(SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/tempDir-001
   [junit4]   2> 172676 INFO  
(SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 172676 INFO  (Thread-343) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 172676 INFO  (Thread-343) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 172776 INFO  
(SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:42837
   [junit4]   2> 172793 INFO  (jetty-launcher-296-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 172793 INFO  (jetty-launcher-296-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 172798 INFO  (jetty-launcher-296-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@69ada8b8{/solr,null,AVAILABLE}
   [junit4]   2> 172800 INFO  (jetty-launcher-296-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@57e4c34c{/solr,null,AVAILABLE}
   [junit4]   2> 172804 INFO  (jetty-launcher-296-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@65df4f79{HTTP/1.1,[http/1.1]}{127.0.0.1:40654}
   [junit4]   2> 172804 INFO  (jetty-launcher-296-thread-1) [    ] 
o.e.j.s.Server Started @175515ms
   [junit4]   2> 172804 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=40654}
   [junit4]   2> 172805 ERROR (jetty-launcher-296-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.0.0
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-01-08T14:22:58.814Z
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@3e5ad244{HTTP/1.1,[http/1.1]}{127.0.0.1:51468}
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-2) [    ] 
o.e.j.s.Server Started @175516ms
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=51468}
   [junit4]   2> 172805 ERROR (jetty-launcher-296-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.0.0
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-01-08T14:22:58.815Z
   [junit4]   2> 172812 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 172812 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 172824 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42837/solr
   [junit4]   2> 172826 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42837/solr
   [junit4]   2> 172837 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x1597e75fc810004, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 172862 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:51468_solr
   [junit4]   2> 172863 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.Overseer Overseer 
(id=97247912177762309-127.0.0.1:51468_solr-n_0000000000) starting
   [junit4]   2> 172879 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:51468_solr
   [junit4]   2> 172882 INFO  
(zkCallback-307-thread-1-processing-n:127.0.0.1:40654_solr) 
[n:127.0.0.1:40654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 172918 INFO  
(OverseerStateUpdate-97247912177762309-127.0.0.1:51468_solr-n_0000000000) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 172919 INFO  (jetty-launcher-296-thread-1) 
[n:127.0.0.1:40654_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:40654_solr
   [junit4]   2> 172922 INFO  
(zkCallback-307-thread-1-processing-n:127.0.0.1:40654_solr) 
[n:127.0.0.1:40654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 172937 INFO  
(zkCallback-306-thread-1-processing-n:127.0.0.1:51468_solr) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 173180 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.CoreContainer Initializing authentication 
plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 173180 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, 
delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, 
zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, 
token.validity=36000, signer.secret.provider.zookeeper.path=/token, 
type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, 
signer.secret.provider=zookeeper}
   [junit4]   2> 173180 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 173182 INFO  (jetty-launcher-296-thread-1) 
[n:127.0.0.1:40654_solr    ] o.a.s.c.CoreContainer Initializing authentication 
plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 173182 INFO  (jetty-launcher-296-thread-1) 
[n:127.0.0.1:40654_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, 
delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, 
zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, 
token.validity=36000, signer.secret.provider.zookeeper.path=/token, 
type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, 
signer.secret.provider=zookeeper}
   [junit4]   2> 173182 INFO  (jetty-launcher-296-thread-1) 
[n:127.0.0.1:40654_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 173196 INFO  (jetty-launcher-296-thread-2-EventThread) 
[n:127.0.0.1:51468_solr    ] o.a.c.f.s.ConnectionStateManager State change: 
CONNECTED
   [junit4]   2> 173197 INFO  (jetty-launcher-296-thread-1-EventThread) 
[n:127.0.0.1:40654_solr    ] o.a.c.f.s.ConnectionStateManager State change: 
CONNECTED
   [junit4]   2> 173267 INFO  (jetty-launcher-296-thread-2) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/tempDir-001/node1/.
   [junit4]   2> 173274 INFO  (jetty-launcher-296-thread-1) 
[n:127.0.0.1:40654_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/tempDir-001/node2/.
   [junit4]   2> 173471 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 173558 ERROR (qtp710559443-1609) [    ] 
o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path 
[/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 173576 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173577 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 
127.0.0.1:42837/solr ready
   [junit4]   2> 173587 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173594 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173597 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 
127.0.0.1:42837/solr ready
   [junit4]   2> 173600 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173630 ERROR (qtp450497154-1614) [    ] 
o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path 
[/ZKDTSMTokensRoot/DT_2]
   [junit4]   2> 173638 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173646 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173646 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 173675 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 173684 WARN  (qtp710559443-1606) [    ] 
o.a.h.s.a.s.AuthenticationFilter Authentication exception: 
org.apache.hadoop.security.AccessControlException: foo tries to renew a token 
with renewer bar
   [junit4]   2> 173685 WARN  (qtp710559443-1606) [    ] o.a.s.s.KerberosPlugin 
Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173685 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173687 WARN  (qtp450497154-1601) [    ] 
o.a.h.s.a.s.AuthenticationFilter Authentication exception: 
org.apache.hadoop.security.AccessControlException: foo tries to renew a token 
with renewer bar
   [junit4]   2> 173688 WARN  (qtp450497154-1601) [    ] o.a.s.s.KerberosPlugin 
Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173688 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173696 WARN  (qtp710559443-1602) [    ] 
o.a.h.s.a.s.AuthenticationFilter Authentication exception: 
org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew 
a token with renewer renewUser
   [junit4]   2> 173697 WARN  (qtp710559443-1602) [    ] o.a.s.s.KerberosPlugin 
Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173697 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173698 WARN  (qtp450497154-1601) [    ] 
o.a.h.s.a.s.AuthenticationFilter Authentication exception: 
org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew 
a token with renewer renewUser
   [junit4]   2> 173699 WARN  (qtp450497154-1601) [    ] o.a.s.s.KerberosPlugin 
Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173699 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173699 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 173752 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 173757 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173759 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 
127.0.0.1:42837/solr ready
   [junit4]   2> 173762 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 173762 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 173762 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth="u=bar&p=bar&t=test&e=1483921379770&s=Fr4VIeIcylBaJOkGhsRiRh2PBaA=";
 Path=/; Domain=127.0.0.1; Expires= , 09-1 -2017 00:22:59 GMT; HttpOnly". 
Invalid 'expires' attribute: , 09-1 -2017 00:22:59 GMT
   [junit4]   2> 173765 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 173796 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 173812 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 173812 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 173819 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173820 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 
127.0.0.1:42837/solr ready
   [junit4]   2> 173824 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173830 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 173830 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 173836 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173837 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 173875 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 173886 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173895 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgEtigFZooKFLQYEFFbqAZ5kWo5jtshoBIcVb-NDr22XB3NvbHItZHQA&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 173895 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgEtigFZooKFLQYEFFbqAZ5kWo5jtshoBIcVb-NDr22XB3NvbHItZHQA&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 173901 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 173924 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 173950 ERROR (qtp450497154-1603) [    ] 
o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path 
[/ZKDTSMTokensRoot/DT_7]
   [junit4]   2> 173954 ERROR (qtp710559443-1608) [    ] 
o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path 
[/ZKDTSMTokensRoot/DT_7]
   [junit4]   2> 173967 ERROR (qtp450497154-1617) [    ] 
o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path 
[/ZKDTSMTokensRoot/DT_8]
   [junit4]   2> 173973 ERROR (qtp710559443-1607) [    ] 
o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path 
[/ZKDTSMTokensRoot/DT_8]
   [junit4]   2> 173974 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> 174000 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 174015 WARN  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] 
o.a.h.s.a.s.AuthenticationFilter Authentication exception: 
org.apache.hadoop.security.AccessControlException: bar is trying to renew a 
token with wrong password
   [junit4]   2> 174015 WARN  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] 
o.a.s.s.KerberosPlugin Could not find 
org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 174017 WARN  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[BD2AC0C81AC84F11])
 [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: 
hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; 
HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 174017 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[BD2AC0C81AC84F11])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestSolrCloudWithDelegationTokens 
-Dtests.method=testDelegationTokenRenew -Dtests.seed=BD2AC0C81AC84F11 
-Dtests.slow=true -Dtests.locale=ko-KR -Dtests.timezone=America/Indiana/Knox 
-Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.05s J1 | 
TestSolrCloudWithDelegationTokens.testDelegationTokenRenew <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but 
was:<403>
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:8AB134D6220492B5]:0)
   [junit4]    >        at 
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.renewDelegationToken(TestSolrCloudWithDelegationTokens.java:130)
   [junit4]    >        at 
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.verifyDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:315)
   [junit4]    >        at 
org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:332)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 174057 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[BD2AC0C81AC84F11]) 
[    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 174083 INFO  
(TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[BD2AC0C81AC84F11]) 
[    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 174084 INFO  (jetty-launcher-296-thread-2) [    ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@65df4f79{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 174085 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=1771798306
   [junit4]   2> 174086 INFO  (jetty-launcher-296-thread-1) [    ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@3e5ad244{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 174086 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=53018233
   [junit4]   2> 174092 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.c.Overseer Overseer 
(id=97247912177762309-127.0.0.1:51468_solr-n_0000000000) closing
   [junit4]   2> 174093 INFO  
(OverseerStateUpdate-97247912177762309-127.0.0.1:51468_solr-n_0000000000) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:51468_solr
   [junit4]   2> 175595 WARN  
(zkCallback-307-thread-1-processing-n:127.0.0.1:40654_solr) 
[n:127.0.0.1:40654_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175595 INFO  (jetty-launcher-296-thread-2) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 175595 WARN  
(zkCallback-306-thread-1-processing-n:127.0.0.1:51468_solr) 
[n:127.0.0.1:51468_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175595 ERROR 
(Thread[Thread-363,5,TGRP-TestSolrCloudWithDelegationTokens]) 
[n:127.0.0.1:40654_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager 
ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 175595 INFO  (jetty-launcher-296-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 175596 ERROR 
(Thread[Thread-360,5,TGRP-TestSolrCloudWithDelegationTokens]) 
[n:127.0.0.1:51468_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager 
ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 175602 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x1597e75fc810008, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 175602 INFO  (jetty-launcher-296-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@57e4c34c{/solr,null,UNAVAILABLE}
   [junit4]   2> 175605 INFO  (jetty-launcher-296-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@69ada8b8{/solr,null,UNAVAILABLE}
   [junit4]   2> 175609 INFO  
(SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:42837 42837
   [junit4]   2> 175631 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxnFactory Ignoring exception
   [junit4]   2> java.nio.channels.ClosedChannelException
   [junit4]   2>        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:235)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:188)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 175632 INFO  (Thread-343) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:42837 42837
   [junit4]   2> 175634 WARN  (Thread-343) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/aliases.json
   [junit4]   2>        6       /solr/clusterprops.json
   [junit4]   2>        2       /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/clusterstate.json
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2>        2       /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/live_nodes
   [junit4]   2>        6       /solr/collections
   [junit4]   2>        3       /solr/overseer/queue
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2>        2       /solr/overseer/queue-work
   [junit4]   2>        2       
/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 175636 INFO  
(SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001
   [junit4]   2> Jan 08, 2017 2:23:01 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=428, maxMBSortInHeap=5.553593308142943, 
sim=RandomSimilarity(queryNorm=true): {}, locale=ko-KR, 
timezone=America/Indiana/Knox
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 
(64-bit)/cpus=3,threads=1,free=40833776,total=197660672
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestRuleBasedAuthorizationPlugin, NoCacheHeaderTest, TestFieldCacheReopen, 
SolrCoreMetricManagerTest, TestConfigSetsAPI, LeaderElectionIntegrationTest, 
PreAnalyzedFieldTest, TestSolrCloudWithSecureImpersonation, 
TestAuthenticationFramework, BadIndexSchemaTest, TestComponentsName, 
ResponseHeaderTest, AsyncCallRequestStatusResponseTest, 
TestSortByMinMaxFunction, SpellPossibilityIteratorTest, TestSchemaResource, 
TestReplicationHandlerBackup, InfoHandlerTest, ClusterStateTest, 
ZkSolrClientTest, TestSimpleTrackingShardHandler, TestTestInjection, 
TestXmlQParser, TestInitQParser, HdfsBasicDistributedZk2Test, TestFastWriter, 
TestJmxMonitoredMap, TemplateUpdateProcessorTest, SolrMetricReporterTest, 
TestSolr4Spatial, TestNoOpRegenerator, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [57/676 (1!)] on J1 in 3.49s, 8 tests, 1 failure <<< 
FAILURES!

[...truncated 55 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsNNFailoverTest
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/init-core-data-001
   [junit4]   2> 214030 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 214030 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 214626 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 215649 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 215959 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via 
org.mortbay.log.Slf4jLog
   [junit4]   2> 215989 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 216080 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 216137 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_solaris.vm_36490_hdfs____.hffxba/webapp
   [junit4]   2> 217800 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:36490
   [junit4]   2> 218874 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 218896 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 218910 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_64144_datanode____.3siw12/webapp
   [junit4]   2> 219593 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:64144
   [junit4]   2> 220092 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 220095 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 220112 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_57301_datanode____.oo31xj/webapp
   [junit4]   2> 221313 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57301
   [junit4]   2> 222105 INFO  (IPC Server handler 5 on 50684) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-973dbff0-4810-4d3f-a6ad-7ec37ebcaa56 node 
DatanodeRegistration(127.0.0.1:46433, 
datanodeUuid=24fd6232-c359-471d-a0f0-d76dbc930d2e, infoPort=33548, 
infoSecurePort=0, ipcPort=63593, 
storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 4 msecs
   [junit4]   2> 222106 INFO  (IPC Server handler 2 on 50684) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-e6a44d72-20a5-4d62-a9f6-91051665d38a node 
DatanodeRegistration(127.0.0.1:38002, 
datanodeUuid=aa35785a-ab9f-4c33-874c-3f53750b86a3, infoPort=52061, 
infoSecurePort=0, ipcPort=61050, 
storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 222106 INFO  (IPC Server handler 5 on 50684) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-3a588cb3-d006-42ec-9aee-924ab9d4c30a node 
DatanodeRegistration(127.0.0.1:46433, 
datanodeUuid=24fd6232-c359-471d-a0f0-d76dbc930d2e, infoPort=33548, 
infoSecurePort=0, ipcPort=63593, 
storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 222129 INFO  (IPC Server handler 2 on 50684) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-7009a6f9-00ed-42ac-97c2-4accfec4317d node 
DatanodeRegistration(127.0.0.1:38002, 
datanodeUuid=aa35785a-ab9f-4c33-874c-3f53750b86a3, infoPort=52061, 
infoSecurePort=0, ipcPort=61050, 
storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 222906 INFO  
(TEST-HdfsNNFailoverTest.test-seed#[BD2AC0C81AC84F11]) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 222906 INFO  (Thread-359) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 222906 INFO  (Thread-359) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 223006 INFO  
(TEST-HdfsNNFailoverTest.test-seed#[BD2AC0C81AC84F11]) [    ] 
o.a.s.c.ZkTestServer start zk server on port:63593
   [junit4]   2> 223009 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:41363 got 
version 0 expected version 9
   [junit4]   2> 224372 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:60391 got 
version 0 expected version 9
   [junit4]   2> 226111 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:44827 got 
version 0 expected version 9
   [junit4]   2> 228205 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:40233 got 
version 0 expected version 9
   [junit4]   2> 229929 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:63452 got 
version 0 expected version 9
   [junit4]   2> 231070 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:45557 got 
version 0 expected version 9
   [junit4]   2> 232555 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:36093 got 
version 0 expected version 9
   [junit4]   2> 233766 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:63200 got 
version 0 expected version 9
   [junit4]   2> 235394 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:57636 got 
version 0 expected version 9
   [junit4]   2> 237244 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:35493 got 
version 0 expected version 9
   [junit4]   2> 238370 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:38704 got 
version 0 expected version 9
   [junit4]   2> 240023 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:35108 got 
version 0 expected version 9
   [junit4]   2> 241809 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:44341 got 
version 0 expected version 9
   [junit4]   2> 243883 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:48548 got 
version 0 expected version 9
   [junit4]   2> 245735 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:45283 got 
version 0 expected version 9
   [junit4]   2> 246862 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:57530 got 
version 0 expected version 9
   [junit4]   2> 248482 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:43575 got 
version 0 expected version 9
   [junit4]   2> 250435 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:62368 got 
version 0 expected version 9
   [junit4]   2> 251862 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:52824 got 
version 0 expected version 9
   [junit4]   2> 253555 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:51417 got 
version 0 expected version 9
   [junit4]   2> 255295 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:52686 got 
version 0 expected version 9
   [junit4]   2> 257095 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:62613 got 
version 0 expected version 9
   [junit4]   2> 258815 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:43324 got 
version 0 expected version 9
   [junit4]   2> 260569 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:34597 got 
version 0 expected version 9
   [junit4]   2> 262557 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:58015 got 
version 0 expected version 9
   [junit4]   2> 264268 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:49334 got 
version 0 expected version 9
   [junit4]   2> 266357 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:64259 got 
version 0 expected version 9
   [junit4]   2> 267627 WARN  (Socket Reader #1 for port 63593) [    ] 
o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:44373 got 
version 0 expected version 9
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsNNFailoverTest 
-Dtests.method=test -Dtests.seed=BD2AC0C81AC84F11 -Dtests.slow=true 
-Dtests.locale=ar-TN -Dtests.timezone=America/Guadeloupe -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   45.6s J0 | HdfsNNFailoverTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: 
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 
127.0.0.1:63593 within 45000 ms
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:357EFF12B43422E9]:0)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
   [junit4]    >        at 
org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:86)
   [junit4]    >        at 
org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:80)
   [junit4]    >        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.distribSetUp(AbstractDistribZkTestBase.java:81)
   [junit4]    >        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribSetUp(AbstractFullDistribZkTestBase.java:219)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:955)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not 
connect to ZooKeeper 127.0.0.1:63593 within 45000 ms
   [junit4]    >        at 
org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
   [junit4]    >        ... 38 more
   [junit4]   2> 268498 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 268508 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 268614 WARN  (DataNode: 
[[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data3/,
 
[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data4/]]
  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode 
BPOfferService for Block pool BP-221125253-127.0.0.1-1483885425298 (Datanode 
Uuid 24fd6232-c359-471d-a0f0-d76dbc930d2e) service to 
solaris-vm/127.0.0.1:50684 interrupted
   [junit4]   2> 268614 WARN  (DataNode: 
[[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data3/,
 
[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data4/]]
  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode 
Ending block pool service for: Block pool BP-221125253-127.0.0.1-1483885425298 
(Datanode Uuid 24fd6232-c359-471d-a0f0-d76dbc930d2e) service to 
solaris-vm/127.0.0.1:50684
   [junit4]   2> 268621 WARN  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 268634 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 268641 WARN  (DataNode: 
[[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data1/,
 
[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data2/]]
  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode 
BPOfferService for Block pool BP-221125253-127.0.0.1-1483885425298 (Datanode 
Uuid aa35785a-ab9f-4c33-874c-3f53750b86a3) service to 
solaris-vm/127.0.0.1:50684 interrupted
   [junit4]   2> 268641 WARN  (DataNode: 
[[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data1/,
 
[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data2/]]
  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode 
Ending block pool service for: Block pool BP-221125253-127.0.0.1-1483885425298 
(Datanode Uuid aa35785a-ab9f-4c33-874c-3f53750b86a3) service to 
solaris-vm/127.0.0.1:50684
   [junit4]   2> 268961 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log 
Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:0
   [junit4]   2> 269072 ERROR 
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
   [junit4]   2> java.lang.reflect.InvocationTargetException
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2>        at 
com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest.teardownClass(HdfsNNFailoverTest.java:48)
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 54 more
   [junit4]   2> 269078 INFO  
(SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001
   [junit4]   2> Jan 08, 2017 2:24:39 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 29 leaked 
thread(s).
   [junit4]   2> Jan 08, 2017 2:24:59 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 5 threads leaked from SUITE scope at 
org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   [junit4]   2>    1) Thread[id=1610, name=SyncThread:0, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   [junit4]   2>    2) Thread[id=1608, 
name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2>         at 
sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2>         at 
sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2>         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2>         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    3) Thread[id=1607, name=Thread-359, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]   2>         at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]   2>         at 
org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]   2>    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, 
state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   [junit4]   2>    5) Thread[id=1609, name=SessionTracker, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at 
org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]   2> Jan 08, 2017 2:24:59 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=1610, name=SyncThread:0, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    2) Thread[id=1608, 
name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    3) Thread[id=1607, name=Thread-359, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, 
state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    5) Thread[id=1609, name=SessionTracker, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2> 289152 ERROR (SyncThread:0) [    ] 
o.a.z.s.SyncRequestProcessor Severe unrecoverable error, exiting
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>        at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   [junit4]   2> 289153 ERROR (SessionTracker) [    ] 
o.a.z.s.SessionTrackerImpl Unexpected interruption
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at java.lang.Object.wait(Native Method)
   [junit4]   2>        at 
org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]   2> 289153 ERROR (ProcessThread(sid:0 cport:-1):) [    ] 
o.a.z.s.PrepRequestProcessor Unexpected interruption
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>        at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   [junit4]   2> 289157 WARN  (Thread-359) [    ] o.a.s.c.ZkTestServer Server 
interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at java.lang.Object.wait(Native Method)
   [junit4]   2>        at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>        at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]   2>        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]   2>        at 
org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]   2> Jan 08, 2017 2:25:02 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be 
terminated:
   [junit4]   2>    1) Thread[id=1608, 
name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native 
Method)
   [junit4]   2>         at 
sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]   2>         at 
sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]   2>         at 
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]   2>         at 
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]   2>         at 
sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]   2>         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2>         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1492, maxMBSortInHeap=6.367441842716158, 
sim=RandomSimilarity(queryNorm=true): {}, locale=ar-TN, 
timezone=America/Guadeloupe
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 
(64-bit)/cpus=3,threads=3,free=84815800,total=254988288
   [junit4]   2> NOTE: All tests run in this JVM: [TestObjectReleaseTracker, 
TestBackupRepositoryFactory, TestBinaryField, PreAnalyzedUpdateProcessorTest, 
TestSolrXml, TestXmlQParserPlugin, TestSolrCloudWithKerberosAlt, 
RecoveryZkTest, CursorPagingTest, TestIBSimilarityFactory, SolrXmlInZkTest, 
SortSpecParsingTest, VersionInfoTest, SolrIndexConfigTest, TestJmxIntegration, 
HttpSolrCallGetCoreTest, TestSort, TestRemoteStreaming, 
QueryElevationComponentTest, TestUpdate, SimplePostToolTest, 
TestSolrCLIRunExample, TestLazyCores, TestLeaderElectionWithEmptyReplica, 
MetricsHandlerTest, TestCustomStream, HdfsNNFailoverTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsNNFailoverTest 
-Dtests.seed=BD2AC0C81AC84F11 -Dtests.slow=true -Dtests.locale=ar-TN 
-Dtests.timezone=America/Guadeloupe -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | HdfsNNFailoverTest (suite) <<<
   [junit4]    > Throwable #1: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE 
scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   [junit4]    >    1) Thread[id=1610, name=SyncThread:0, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   [junit4]    >    2) Thread[id=1608, 
name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]    >         at 
sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]    >         at 
sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    3) Thread[id=1607, name=Thread-359, state=WAITING, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1249)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1323)
   [junit4]    >         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]    >         at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]    >         at 
org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]    >    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, 
state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   [junit4]    >    5) Thread[id=1609, name=SessionTracker, 
state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=1608, 
name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native 
Method)
   [junit4]    >         at 
sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]    >         at 
sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]    >         at 
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at 
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at 
sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)
   [junit4] Completed [77/676 (2!)] on J0 in 78.20s, 1 test, 3 errors <<< 
FAILURES!

[...truncated 64357 lines...]

---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to