Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2026/

1 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; 
[core_node2:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:35279/pm_n","node_name":"127.0.0.1:35279_pm_n","state":"active","type":"TLOG","leader":"true"}];
 clusterState: 
DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={   
"pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{         
"core_node1":{           "state":"down",           
"base_url":"http://127.0.0.1:45574/pm_n";,           
"core":"c8n_1x3_lf_shard1_replica_t1",           
"node_name":"127.0.0.1:45574_pm_n",           "type":"TLOG"},         
"core_node2":{           "core":"c8n_1x3_lf_shard1_replica_t2",           
"base_url":"http://127.0.0.1:35279/pm_n";,           
"node_name":"127.0.0.1:35279_pm_n",           "state":"active",           
"type":"TLOG",           "leader":"true"},         "core_node3":{           
"core":"c8n_1x3_lf_shard1_replica_t3",           
"base_url":"http://127.0.0.1:38161/pm_n";,           
"node_name":"127.0.0.1:38161_pm_n",           "state":"down",           
"type":"TLOG"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1", 
  "autoAddReplicas":"false",   "nrtReplicas":"0",   "tlogReplicas":"3"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 
1; 
[core_node2:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:35279/pm_n","node_name":"127.0.0.1:35279_pm_n","state":"active","type":"TLOG","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:45574/pm_n";,
          "core":"c8n_1x3_lf_shard1_replica_t1",
          "node_name":"127.0.0.1:45574_pm_n",
          "type":"TLOG"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica_t2",
          "base_url":"http://127.0.0.1:35279/pm_n";,
          "node_name":"127.0.0.1:35279_pm_n",
          "state":"active",
          "type":"TLOG",
          "leader":"true"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica_t3",
          "base_url":"http://127.0.0.1:38161/pm_n";,
          "node_name":"127.0.0.1:38161_pm_n",
          "state":"down",
          "type":"TLOG"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"0",
  "tlogReplicas":"3"}
        at 
__randomizedtesting.SeedInfo.seed([35C19711525930E5:BD95A8CBFCA55D1D]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12549 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> 2840797 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[35C19711525930E5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/init-core-data-001
   [junit4]   2> 2840798 WARN  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[35C19711525930E5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=25 numCloses=25
   [junit4]   2> 2840798 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[35C19711525930E5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2840799 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[35C19711525930E5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 2840799 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[35C19711525930E5]-worker) [    ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /pm_n/
   [junit4]   2> 2840822 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2840822 INFO  (Thread-5599) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2840822 INFO  (Thread-5599) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 2840826 ERROR (Thread-5599) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2840923 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ZkTestServer start zk server on port:46555
   [junit4]   2> 2840946 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 2840947 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/schema.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 2840948 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2840949 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 2840949 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 2840950 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 2840951 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2840952 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2840952 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2840953 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2840954 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 2840955 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly 
asked otherwise
   [junit4]   2> 2841034 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2841035 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3a00737f{/pm_n,null,AVAILABLE}
   [junit4]   2> 2841035 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@1e943882{HTTP/1.1,[http/1.1]}{127.0.0.1:44543}
   [junit4]   2> 2841035 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server Started @2849543ms
   [junit4]   2> 2841038 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/tempDir-001/control/data,
 replicaType=NRT, hostContext=/pm_n, hostPort=45379, 
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/control-001/cores}
   [junit4]   2> 2841038 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2841038 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 2841038 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2841038 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 2841038 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-07-22T17:56:52.512Z
   [junit4]   2> 2841040 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 2841040 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/control-001/solr.xml
   [junit4]   2> 2841044 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 2841062 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46555/solr
   [junit4]   2> 2841134 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2841135 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:45379_pm_n
   [junit4]   2> 2841135 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.Overseer Overseer 
(id=98352903769227268-127.0.0.1:45379_pm_n-n_0000000000) starting
   [junit4]   2> 2841139 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:45379_pm_n
   [junit4]   2> 2841143 INFO  
(zkCallback-2116-thread-1-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2841251 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2841255 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2841255 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2841256 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/control-001/cores
   [junit4]   2> 2841284 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2841284 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46555/solr ready
   [junit4]   2> 2841284 INFO  (SocketProxy-Acceptor-45379) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=54508,localport=45379], receiveBufferSize:531000
   [junit4]   2> 2841285 INFO  (SocketProxy-Acceptor-45379) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=44543,localport=45876], receiveBufferSize=530904
   [junit4]   2> 2841290 INFO  (qtp149292787-17455) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45379_pm_n&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2841293 INFO  
(OverseerThreadFactory-6147-thread-1-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.CreateCollectionCmd Create collection 
control_collection
   [junit4]   2> 2841396 INFO  (SocketProxy-Acceptor-45379) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=54512,localport=45379], receiveBufferSize:531000
   [junit4]   2> 2841400 INFO  (SocketProxy-Acceptor-45379) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=44543,localport=45880], receiveBufferSize=530904
   [junit4]   2> 2841402 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2841402 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 2841506 INFO  
(zkCallback-2116-thread-1-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/control_collection/state.json] for collection 
[control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2842434 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2842513 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2842721 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2842739 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' 
using configuration from collection control_collection, trusted=true
   [junit4]   2> 2842740 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.control_collection.shard1.replica_n1' (registry 
'solr.core.control_collection.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2842740 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2842740 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore 
at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/control-001/cores/control_collection_shard1_replica_n1],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 2842742 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=1.0]
   [junit4]   2> 2842744 WARN  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 2842831 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 2842831 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2842832 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2842832 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2842833 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.19037060021490726]
   [junit4]   2> 2842833 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@2312a1e4[control_collection_shard1_replica_n1] main]
   [junit4]   2> 2842834 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 2842834 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2842835 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2842854 INFO  
(searcherExecutor-6150-thread-1-processing-n:127.0.0.1:45379_pm_n 
x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) 
[n:127.0.0.1:45379_pm_n c:control_collection s:shard1  
x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore 
[control_collection_shard1_replica_n1] Registered new searcher 
Searcher@2312a1e4[control_collection_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2842854 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1573646462435196928
   [junit4]   2> 2842859 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2842859 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2842859 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:45379/pm_n/control_collection_shard1_replica_n1/
   [junit4]   2> 2842859 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2842859 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SyncStrategy 
http://127.0.0.1:45379/pm_n/control_collection_shard1_replica_n1/ has no 
replicas
   [junit4]   2> 2842859 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2842861 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:45379/pm_n/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 2842969 INFO  
(zkCallback-2116-thread-1-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/control_collection/state.json] for collection 
[control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2843012 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2843014 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=1611
   [junit4]   2> 2843019 INFO  (qtp149292787-17455) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 2843120 INFO  
(zkCallback-2116-thread-1-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/control_collection/state.json] for collection 
[control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2843302 INFO  
(OverseerCollectionConfigSetProcessor-98352903769227268-127.0.0.1:45379_pm_n-n_0000000000)
 [n:127.0.0.1:45379_pm_n    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2844020 INFO  (qtp149292787-17455) [n:127.0.0.1:45379_pm_n    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45379_pm_n&wt=javabin&version=2}
 status=0 QTime=2729
   [junit4]   2> 2844034 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2844034 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46555/solr ready
   [junit4]   2> 2844034 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection 
loss:false
   [junit4]   2> 2844035 INFO  (SocketProxy-Acceptor-45379) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=54526,localport=45379], receiveBufferSize:531000
   [junit4]   2> 2844035 INFO  (SocketProxy-Acceptor-45379) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=44543,localport=45894], receiveBufferSize=530904
   [junit4]   2> 2844039 INFO  (qtp149292787-17453) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2844058 INFO  
(OverseerThreadFactory-6147-thread-2-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.CreateCollectionCmd Create collection 
collection1
   [junit4]   2> 2844058 WARN  
(OverseerThreadFactory-6147-thread-2-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.CreateCollectionCmd It is unusual to 
create a collection (collection1) without cores.
   [junit4]   2> 2844327 INFO  (qtp149292787-17453) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 2844327 INFO  (qtp149292787-17453) [n:127.0.0.1:45379_pm_n    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2}
 status=0 QTime=288
   [junit4]   2> 2844647 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-1-001
 of type TLOG
   [junit4]   2> 2844655 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2844656 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6d5f86b9{/pm_n,null,AVAILABLE}
   [junit4]   2> 2844656 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@22783eaf{HTTP/1.1,[http/1.1]}{127.0.0.1:38490}
   [junit4]   2> 2844656 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server Started @2853164ms
   [junit4]   2> 2844659 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/tempDir-001/jetty1,
 replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pm_n, 
hostPort=35279, 
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-1-001/cores}
   [junit4]   2> 2844659 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2844659 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 2844659 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2844659 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 2844659 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-07-22T17:56:56.133Z
   [junit4]   2> 2844661 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 2844661 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-1-001/solr.xml
   [junit4]   2> 2844665 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 2844667 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46555/solr
   [junit4]   2> 2844685 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2844687 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2844688 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:35279_pm_n
   [junit4]   2> 2844689 INFO  
(zkCallback-2116-thread-1-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2844710 INFO  (zkCallback-2123-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2844751 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2844869 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2844873 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2844873 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2844874 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-1-001/cores
   [junit4]   2> 2844914 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with 
params 
node=127.0.0.1:35279_pm_n&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2844920 INFO  
(OverseerCollectionConfigSetProcessor-98352903769227268-127.0.0.1:45379_pm_n-n_0000000000)
 [n:127.0.0.1:45379_pm_n    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2844920 INFO  
(OverseerThreadFactory-6147-thread-3-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.AddReplicaCmd Node Identified 
127.0.0.1:35279_pm_n for creating new replica
   [junit4]   2> 2844921 INFO  (SocketProxy-Acceptor-35279) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=38356,localport=35279], receiveBufferSize:531000
   [junit4]   2> 2844922 INFO  (SocketProxy-Acceptor-35279) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=38490,localport=46472], receiveBufferSize=530904
   [junit4]   2> 2844941 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2844941 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 2845044 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2845954 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2845995 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.s.IndexSchema 
[collection1_shard2_replica_t1] Schema name=test
   [junit4]   2> 2846237 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2846259 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1_shard2_replica_t1' using configuration from 
collection collection1, trusted=true
   [junit4]   2> 2846259 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.collection1.shard2.replica_t1' (registry 
'solr.core.collection1.shard2.replica_t1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2846259 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 2846259 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SolrCore 
[[collection1_shard2_replica_t1] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-1-001/cores/collection1_shard2_replica_t1],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-1-001/cores/collection1_shard2_replica_t1/data/]
   [junit4]   2> 2846274 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=1.0]
   [junit4]   2> 2846301 WARN  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 2846380 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2846380 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2846381 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2846381 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2846398 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.19037060021490726]
   [junit4]   2> 2846398 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@17370199[collection1_shard2_replica_t1] main]
   [junit4]   2> 2846399 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 2846400 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2846400 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2846409 INFO  
(searcherExecutor-6161-thread-1-processing-n:127.0.0.1:35279_pm_n 
x:collection1_shard2_replica_t1 s:shard2 c:collection1) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SolrCore 
[collection1_shard2_replica_t1] Registered new searcher 
Searcher@17370199[collection1_shard2_replica_t1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2846409 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1573646466162884608
   [junit4]   2> 2846414 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2846414 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2846414 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:35279/pm_n/collection1_shard2_replica_t1/
   [junit4]   2> 2846414 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SyncStrategy 
Sync Success - now sync replicas to me
   [junit4]   2> 2846414 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.SyncStrategy 
http://127.0.0.1:35279/pm_n/collection1_shard2_replica_t1/ has no replicas
   [junit4]   2> 2846414 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2846430 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.ZkController 
collection1_shard2_replica_t1 stopping background replication from leader
   [junit4]   2> 2846433 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:35279/pm_n/collection1_shard2_replica_t1/ shard2
   [junit4]   2> 2846541 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2846597 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 2846598 INFO  (qtp534745923-17503) [n:127.0.0.1:35279_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=1657
   [junit4]   2> 2846599 INFO  (qtp149292787-17457) [n:127.0.0.1:45379_pm_n    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={node=127.0.0.1:35279_pm_n&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2}
 status=0 QTime=1685
   [junit4]   2> 2846703 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2846792 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-2-001
 of type TLOG
   [junit4]   2> 2846800 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2846801 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1fb88f8a{/pm_n,null,AVAILABLE}
   [junit4]   2> 2846801 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@30cbc359{HTTP/1.1,[http/1.1]}{127.0.0.1:43885}
   [junit4]   2> 2846802 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server Started @2855309ms
   [junit4]   2> 2846804 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/tempDir-001/jetty2,
 replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pm_n, 
hostPort=38161, 
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-2-001/cores}
   [junit4]   2> 2846804 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2846806 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 2846806 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2846806 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 2846806 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-07-22T17:56:58.280Z
   [junit4]   2> 2846808 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 2846808 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-2-001/solr.xml
   [junit4]   2> 2846812 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 2846826 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46555/solr
   [junit4]   2> 2846832 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 2846834 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2846835 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:38161_pm_n
   [junit4]   2> 2846836 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 2846836 INFO  
(zkCallback-2116-thread-2-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 2846840 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 2846840 INFO  (zkCallback-2123-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2846926 INFO  
(OverseerCollectionConfigSetProcessor-98352903769227268-127.0.0.1:45379_pm_n-n_0000000000)
 [n:127.0.0.1:45379_pm_n    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2847020 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2847033 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2847033 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2847034 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-2-001/cores
   [junit4]   2> 2847097 INFO  (qtp149292787-17455) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with 
params 
node=127.0.0.1:38161_pm_n&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2847099 INFO  
(OverseerThreadFactory-6147-thread-4-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.AddReplicaCmd Node Identified 
127.0.0.1:38161_pm_n for creating new replica
   [junit4]   2> 2847101 INFO  (SocketProxy-Acceptor-38161) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=54466,localport=38161], receiveBufferSize:531000
   [junit4]   2> 2847120 INFO  (SocketProxy-Acceptor-38161) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=43885,localport=43646], receiveBufferSize=530904
   [junit4]   2> 2847123 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2847124 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 2847245 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 2847245 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 2848139 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2848169 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.s.IndexSchema 
[collection1_shard1_replica_t3] Schema name=test
   [junit4]   2> 2848413 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2848431 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1_shard1_replica_t3' using configuration from 
collection collection1, trusted=true
   [junit4]   2> 2848432 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.collection1.shard1.replica_t3' (registry 
'solr.core.collection1.shard1.replica_t3') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2848432 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 2848432 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SolrCore 
[[collection1_shard1_replica_t3] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-2-001/cores/collection1_shard1_replica_t3],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-2-001/cores/collection1_shard1_replica_t3/data/]
   [junit4]   2> 2848434 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=1.0]
   [junit4]   2> 2848436 WARN  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 2848515 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2848515 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2848516 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2848516 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2848517 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.19037060021490726]
   [junit4]   2> 2848517 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@213c99ee[collection1_shard1_replica_t3] main]
   [junit4]   2> 2848518 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 2848518 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2848527 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2848535 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1573646468383768576
   [junit4]   2> 2848554 INFO  
(searcherExecutor-6172-thread-1-processing-n:127.0.0.1:38161_pm_n 
x:collection1_shard1_replica_t3 s:shard1 c:collection1) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SolrCore 
[collection1_shard1_replica_t3] Registered new searcher 
Searcher@213c99ee[collection1_shard1_replica_t3] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2848563 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2848563 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2848563 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:38161/pm_n/collection1_shard1_replica_t3/
   [junit4]   2> 2848563 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SyncStrategy 
Sync Success - now sync replicas to me
   [junit4]   2> 2848563 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.SyncStrategy 
http://127.0.0.1:38161/pm_n/collection1_shard1_replica_t3/ has no replicas
   [junit4]   2> 2848563 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2848564 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.ZkController 
collection1_shard1_replica_t3 stopping background replication from leader
   [junit4]   2> 2848565 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:38161/pm_n/collection1_shard1_replica_t3/ shard1
   [junit4]   2> 2848684 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 2848684 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 2848741 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 2848742 INFO  (qtp1736140296-17536) [n:127.0.0.1:38161_pm_n 
c:collection1 s:shard1  x:collection1_shard1_replica_t3] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=1618
   [junit4]   2> 2848755 INFO  (qtp149292787-17455) [n:127.0.0.1:45379_pm_n    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={node=127.0.0.1:38161_pm_n&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2}
 status=0 QTime=1658
   [junit4]   2> 2848860 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 2848860 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 2849003 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-3-001
 of type TLOG
   [junit4]   2> 2849004 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2849004 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1127ad3b{/pm_n,null,AVAILABLE}
   [junit4]   2> 2849005 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@206e80d3{HTTP/1.1,[http/1.1]}{127.0.0.1:34084}
   [junit4]   2> 2849020 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.e.j.s.Server Started @2857528ms
   [junit4]   2> 2849020 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/tempDir-001/jetty3,
 replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pm_n, 
hostPort=45574, 
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-3-001/cores}
   [junit4]   2> 2849020 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2849022 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 2849022 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2849022 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 2849023 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-07-22T17:57:00.497Z
   [junit4]   2> 2849025 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 2849025 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-3-001/solr.xml
   [junit4]   2> 2849037 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 2849052 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46555/solr
   [junit4]   2> 2849058 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (3)
   [junit4]   2> 2849067 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2849069 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:45574_pm_n
   [junit4]   2> 2849069 INFO  
(zkCallback-2116-thread-2-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2849072 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2849093 INFO  
(zkCallback-2140-thread-1-processing-n:127.0.0.1:45574_pm_n) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2849100 INFO  
(OverseerCollectionConfigSetProcessor-98352903769227268-127.0.0.1:45379_pm_n-n_0000000000)
 [n:127.0.0.1:45379_pm_n    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2849103 INFO  (zkCallback-2123-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2849107 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2849207 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2849211 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2849211 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2849220 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-3-001/cores
   [junit4]   2> 2849267 INFO  (qtp149292787-17456) [n:127.0.0.1:45379_pm_n    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with 
params 
node=127.0.0.1:45574_pm_n&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2849269 INFO  
(OverseerThreadFactory-6147-thread-5-processing-n:127.0.0.1:45379_pm_n) 
[n:127.0.0.1:45379_pm_n    ] o.a.s.c.AddReplicaCmd Node Identified 
127.0.0.1:45574_pm_n for creating new replica
   [junit4]   2> 2849287 INFO  (SocketProxy-Acceptor-45574) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=37148,localport=45574], receiveBufferSize:531000
   [junit4]   2> 2849287 INFO  (SocketProxy-Acceptor-45574) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=34084,localport=56182], receiveBufferSize=530904
   [junit4]   2> 2849310 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t5&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2849310 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 2849427 INFO  
(zkCallback-2140-thread-1-processing-n:127.0.0.1:45574_pm_n) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 2849427 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 2849427 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 2850335 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2850349 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.s.IndexSchema 
[collection1_shard2_replica_t5] Schema name=test
   [junit4]   2> 2850510 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2850542 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1_shard2_replica_t5' using configuration from 
collection collection1, trusted=true
   [junit4]   2> 2850542 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.collection1.shard2.replica_t5' (registry 
'solr.core.collection1.shard2.replica_t5') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@30ccc941
   [junit4]   2> 2850542 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 2850543 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.c.SolrCore 
[[collection1_shard2_replica_t5] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-3-001/cores/collection1_shard2_replica_t5],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001/shard-3-001/cores/collection1_shard2_replica_t5/data/]
   [junit4]   2> 2850544 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=1.0]
   [junit4]   2> 2850546 WARN  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 2850685 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2850685 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2850686 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2850686 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2850699 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.19037060021490726]
   [junit4]   2> 2850700 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@187ab9e5[collection1_shard2_replica_t5] main]
   [junit4]   2> 2850700 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 2850701 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2850701 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2850705 INFO  
(searcherExecutor-6183-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.c.SolrCore 
[collection1_shard2_replica_t5] Registered new searcher 
Searcher@187ab9e5[collection1_shard2_replica_t5] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2850705 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1573646470667567104
   [junit4]   2> 2850710 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.c.ZkController 
Core needs to recover:collection1_shard2_replica_t5
   [junit4]   2> 2850719 INFO  
(updateExecutor-2137-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] 
o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 2850719 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.c.RecoveryStrategy Starting recovery 
process. recoveringAfterStartup=true
   [junit4]   2> 2850719 INFO  (qtp2129806690-17568) [n:127.0.0.1:45574_pm_n 
c:collection1 s:shard2  x:collection1_shard2_replica_t5] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t5&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=1409
   [junit4]   2> 2850722 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.c.RecoveryStrategy ###### 
startupVersions=[[]]
   [junit4]   2> 2850722 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.c.ZkController 
collection1_shard2_replica_t5 stopping background replication from leader
   [junit4]   2> 2850722 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.c.RecoveryStrategy Begin buffering 
updates. core=[collection1_shard2_replica_t5]
   [junit4]   2> 2850722 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.u.UpdateLog Starting to buffer updates. 
FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 2850722 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.c.RecoveryStrategy Publishing state of 
core [collection1_shard2_replica_t5] as recovering, leader is 
[http://127.0.0.1:35279/pm_n/collection1_shard2_replica_t1/] and I am 
[http://127.0.0.1:45574/pm_n/collection1_shard2_replica_t5/]
   [junit4]   2> 2850732 INFO  
(recoveryExecutor-2138-thread-1-processing-n:127.0.0.1:45574_pm_n 
x:collection1_shard2_replica_t5 s:shard2 c:collection1 r:core_node6) 
[n:127.0.0.1:45574_pm_n c:collection1 s:shard2 r:core_node6 
x:collection1_shard2_replica_t5] o.a.s.c.RecoveryStrategy Sending prep recovery 
command to [http://127.0.0.1:35279/pm_n]; [WaitForState: 
action=PREPRECOVERY&core=collection1_shard2_replica_t1&nodeName=127.0.0.1:45574_pm_n&coreNodeName=core_node6&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 2850732 INFO  (SocketProxy-Acceptor-35279) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=38410,localport=35279], receiveBufferSize:531000
   [junit4]   2> 2850735 INFO  (SocketProxy-Acceptor-35279) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=38490,localport=46526], receiveBufferSize=530904
   [junit4]   2> 2850764 INFO  (qtp534745923-17505) [n:127.0.0.1:35279_pm_n    
] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node6, state: 
recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, 
maxTime: 183 s
   [junit4]   2> 2850764 INFO  (qtp534745923-17505) [n:127.0.0.1:35279_pm_n    
] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, 
shard=shard2, thisCore=collection1_shard2_replica_t1, 
leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, 
currentState=down, localState=active, nodeName=127.0.0.1:45574_pm_n, 
coreNodeName=core_node6, onlyIfActiveCheckResult=false, nodeProps: 
core_node6:{"core":"collection1_shard2_replica_t5","base_url":"http://127.0.0.1:45574/pm_n","node_name":"127.0.0.1:45574_pm_n","state":"down","type":"TLOG"}
   [junit4]   2> 2850764 INFO  (qtp149292787-17456) [n:127.0.0.1:45379_pm_n    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={node=127.0.0.1:45574_pm_n&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2}
 status=0 QTime=1497
   [junit4]   2> 2850767 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 2850767 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 
30000 for each attempt
   [junit4]   2> 2850767 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: 
collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 2850867 INFO  
(zkCallback-2134-thread-1-processing-n:127.0.0.1:38161_pm_n) 
[n:127.0.0.1:38161_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 2850867 INFO  
(zkCallback-2140-thread-1-processing-n:127.0.0.1:45574_pm_n) 
[n:127.0.0.1:45574_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 2850873 INFO  
(zkCallback-2128-thread-1-processing-n:127.0.0.1:35279_pm_n) 
[n:127.0.0.1:35279_pm_n    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 2851269 INFO  
(OverseerCollectionConfigSetProcessor-98352903769227268-127.0.0.1:45379_pm_n-n_0000000000)
 [n:127.0.0.1:45379_pm_n    ] o.a.s.c.Oversee

[...truncated too long message...]

r/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/clusterprops.json
   [junit4]   2>        6       /solr/clusterstate.json
   [junit4]   2>        3       /solr/collections/collection1/state.json
   [junit4]   2>        3       /solr/collections/c8n_1x3_lf/state.json
   [junit4]   2>        2       
/solr/overseer_elect/election/98352903769227273-127.0.0.1:35279_pm_n-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/live_nodes
   [junit4]   2>        6       /solr/collections
   [junit4]   2>        3       /solr/overseer/queue
   [junit4]   2>        3       /solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> 2979212 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SocketProxy Closing 3 connections to: http://127.0.0.1:45379/pm_n, 
target: http://127.0.0.1:44543/pm_n
   [junit4]   2> 2979212 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SocketProxy Closing 6 connections to: http://127.0.0.1:35279/pm_n, 
target: http://127.0.0.1:38490/pm_n
   [junit4]   2> 2979212 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SocketProxy Closing 0 connections to: http://127.0.0.1:38161/pm_n, 
target: http://127.0.0.1:43885/pm_n
   [junit4]   2> 2979212 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[35C19711525930E5]) [    ] 
o.a.s.c.SocketProxy Closing 0 connections to: http://127.0.0.1:45574/pm_n, 
target: http://127.0.0.1:34084/pm_n
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test 
-Dtests.seed=35C19711525930E5 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=en -Dtests.timezone=NZ -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] FAILURE  138s J2 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Expected 2 of 3 
replicas to be active but only found 1; 
[core_node2:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:35279/pm_n","node_name":"127.0.0.1:35279_pm_n","state":"active","type":"TLOG","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={
   [junit4]    >   "pullReplicas":"0",
   [junit4]    >   "replicationFactor":"1",
   [junit4]    >   "shards":{"shard1":{
   [junit4]    >       "range":"80000000-7fffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node1":{
   [junit4]    >           "state":"down",
   [junit4]    >           "base_url":"http://127.0.0.1:45574/pm_n";,
   [junit4]    >           "core":"c8n_1x3_lf_shard1_replica_t1",
   [junit4]    >           "node_name":"127.0.0.1:45574_pm_n",
   [junit4]    >           "type":"TLOG"},
   [junit4]    >         "core_node2":{
   [junit4]    >           "core":"c8n_1x3_lf_shard1_replica_t2",
   [junit4]    >           "base_url":"http://127.0.0.1:35279/pm_n";,
   [junit4]    >           "node_name":"127.0.0.1:35279_pm_n",
   [junit4]    >           "state":"active",
   [junit4]    >           "type":"TLOG",
   [junit4]    >           "leader":"true"},
   [junit4]    >         "core_node3":{
   [junit4]    >           "core":"c8n_1x3_lf_shard1_replica_t3",
   [junit4]    >           "base_url":"http://127.0.0.1:38161/pm_n";,
   [junit4]    >           "node_name":"127.0.0.1:38161_pm_n",
   [junit4]    >           "state":"down",
   [junit4]    >           "type":"TLOG"}}}},
   [junit4]    >   "router":{"name":"compositeId"},
   [junit4]    >   "maxShardsPerNode":"1",
   [junit4]    >   "autoAddReplicas":"false",
   [junit4]    >   "nrtReplicas":"0",
   [junit4]    >   "tlogReplicas":"3"}
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([35C19711525930E5:BD95A8CBFCA55D1D]:0)
   [junit4]    >        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
   [junit4]    >        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_35C19711525930E5-001
   [junit4]   2> Jul 22, 2017 5:59:10 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, 
sim=RandomSimilarity(queryNorm=false): {}, locale=en, timezone=NZ
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 
1.8.0_131 (64-bit)/cpus=4,threads=1,free=93888584,total=526385152
   [junit4]   2> NOTE: All tests run in this JVM: [SmileWriterTest, 
TestSchemaNameResource, TestFastWriter, TestSchemaManager, 
TestFoldingMultitermQuery, BlobRepositoryCloudTest, TestSolrCoreProperties, 
MultiThreadedOCPTest, PingRequestHandlerTest, TestUnifiedSolrHighlighter, 
TestManagedResource, CircularListTest, TestConfigsApi, 
TestFieldCacheVsDocValues, UnloadDistributedZkTest, TestCustomSort, 
HdfsRecoverLeaseTest, BlockDirectoryTest, TestSimpleQParserPlugin, 
ImplicitSnitchTest, CoreMergeIndexesAdminHandlerTest, BlockJoinFacetSimpleTest, 
DeleteInactiveReplicaTest, TestExclusionRuleCollectionAccess, 
TestReversedWildcardFilterFactory, TolerantUpdateProcessorTest, 
TestCollapseQParserPlugin, DistributedFacetPivotWhiteBoxTest, TestLockTree, 
PrimitiveFieldTypeTest, ShardSplitTest, TestRecoveryHdfs, TestReload, 
ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestManagedSchemaAPI, 
DocumentBuilderTest, PolyFieldTest, FastVectorHighlighterTest, 
TestMinMaxOnMultiValuedField, TestDeleteCollectionOnDownNodes, TestBlobHandler, 
TestExpandComponent, TestRuleBasedAuthorizationPlugin, PreAnalyzedFieldTest, 
TestQueryUtils, TestCloudInspectUtil, SolrXmlInZkTest, BasicDistributedZk2Test, 
TestLRUStatsCache, TestImpersonationWithHadoopAuth, 
TestBackupRepositoryFactory, TestClusterStateMutator, TestStressReorder, 
TestSort, TestStressUserVersions, DistribCursorPagingTest, BadIndexSchemaTest, 
CollectionsAPISolrJTest, URLClassifyProcessorTest, ReplicaListTransformerTest, 
CursorMarkTest, TestGroupingSearch, V2StandaloneTest, SortByFunctionTest, 
TestPerFieldSimilarityWithDefaultOverride, SolrCoreCheckLockOnStartupTest, 
TestManagedStopFilterFactory, AliasIntegrationTest, AutoCommitTest, 
TestConfigSetProperties, TestFieldCacheReopen, HdfsUnloadDistributedZkTest, 
TestSolrCloudWithSecureImpersonation, TestJsonRequest, 
TestSolrConfigHandlerConcurrent, TestReplicaProperties, DateMathParserTest, 
HardAutoCommitTest, PropertiesRequestHandlerTest, TestCloudManagedSchema, 
TestDynamicFieldCollectionResource, MetricsConfigTest, 
TestSchemaVersionResource, DistributedFacetPivotLongTailTest, 
TestCloudPseudoReturnFields, BitVectorTest, RegexBoostProcessorTest, 
TestGraphTermsQParserPlugin, EnumFieldTest, TestUseDocValuesAsStored2, 
AddBlockUpdateTest, LeaderElectionTest, TestTolerantUpdateProcessorRandomCloud, 
TestPayloadCheckQParserPlugin, TestJettySolrRunner, CleanupOldIndexTest, 
SolrCloudReportersTest, TestRandomFlRTGCloud, SimpleCollectionCreateDeleteTest, 
TestConfigSetsAPIExclusivity, TestBulkSchemaAPI, TestUtils, 
TestSizeLimitedDistributedMap, DataDrivenBlockJoinTest, 
VMParamsZkACLAndCredentialsProvidersTest, WrapperMergePolicyFactoryTest, 
TestInfoStreamLogging, TestNumericTerms32, MetricUtilsTest, 
OverriddenZkACLAndCredentialsProvidersTest, TestXmlQParserPlugin, 
TestPseudoReturnFields, TestCustomStream, TestLFUCache, SimpleFacetsTest, 
StatsComponentTest, SpellCheckComponentTest, SolrCmdDistributorTest, 
PeerSyncTest, BasicFunctionalityTest, DirectUpdateHandlerTest, 
TestIndexSearcher, ShowFileRequestHandlerTest, SolrIndexSplitterTest, 
SimplePostToolTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, 
CoreAdminHandlerTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, 
TestTrie, SpatialFilterTest, TestCSVLoader, NoCacheHeaderTest, 
SchemaVersionSpecificBehaviorTest, QueryEqualityTest, 
DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, 
DocValuesMultiTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, 
XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, 
IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, 
DirectSolrSpellCheckerTest, FileBasedSpellCheckerTest, TestValueSourceCache, 
TermVectorComponentTest, RequiredFieldsTest, TestJmxIntegration, 
UpdateRequestProcessorFactoryTest, MBeansHandlerTest, 
UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, 
HighlighterConfigTest, SOLR749Test, TestQuerySenderListener, 
AlternateDirectoryTest, TestSolrIndexConfig, CopyFieldTest, 
ResponseLogComponentTest, SolrIndexConfigTest, TestSearchPerf, 
MinimalSchemaTest, OutputWriterTest, NotRequiredUniqueKeyTest, EchoParamsTest, 
TestDFRSimilarityFactory, TestPerFieldSimilarity, TestIBSimilarityFactory, 
ResourceLoaderTest, PrimUtilsTest, SpellingQueryConverterTest, 
RAMDirectoryFactoryTest, ClusterStateTest, TestRTGBase, 
DistributedIntervalFacetingTest, TestHighlightDedupGrouping, 
TestTolerantSearch, TestEmbeddedSolrServerConstructors, 
TestEmbeddedSolrServerSchemaAPI, AssignTest, 
ChaosMonkeySafeLeaderWithPullReplicasTest, 
CollectionsAPIAsyncDistributedZkTest, DeleteLastCustomShardedReplicaTest, 
DeleteNodeTest, DeleteReplicaTest, DistributedQueueTest, ForceLeaderTest, 
LeaderFailoverAfterPartitionTest]
   [junit4] Completed [531/727 (1!)] on J2 in 138.95s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 43926 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to