Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/334/

2 tests failed.
FAILED:  org.apache.solr.cloud.TestCloudRecovery2.test

Error Message:
 Timeout waiting to see state for collection=collection1 
:DocCollection(collection1//collections/collection1/state.json/18)={   
"pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{         
"core_node3":{           "core":"collection1_shard1_replica_n1",           
"base_url":"http://127.0.0.1:38327/solr";,           
"node_name":"127.0.0.1:38327_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node4":{           "core":"collection1_shard1_replica_n2",         
  "base_url":"http://127.0.0.1:35099/solr";,           
"node_name":"127.0.0.1:35099_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"2",   
"autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"} Live 
Nodes: [127.0.0.1:35099_solr, 127.0.0.1:38327_solr] Last available state: 
DocCollection(collection1//collections/collection1/state.json/18)={   
"pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{         
"core_node3":{           "core":"collection1_shard1_replica_n1",           
"base_url":"http://127.0.0.1:38327/solr";,           
"node_name":"127.0.0.1:38327_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node4":{           "core":"collection1_shard1_replica_n2",         
  "base_url":"http://127.0.0.1:35099/solr";,           
"node_name":"127.0.0.1:35099_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"2",   
"autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: 
Timeout waiting to see state for collection=collection1 
:DocCollection(collection1//collections/collection1/state.json/18)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"collection1_shard1_replica_n1",
          "base_url":"http://127.0.0.1:38327/solr";,
          "node_name":"127.0.0.1:38327_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node4":{
          "core":"collection1_shard1_replica_n2",
          "base_url":"http://127.0.0.1:35099/solr";,
          "node_name":"127.0.0.1:35099_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:35099_solr, 127.0.0.1:38327_solr]
Last available state: 
DocCollection(collection1//collections/collection1/state.json/18)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"collection1_shard1_replica_n1",
          "base_url":"http://127.0.0.1:38327/solr";,
          "node_name":"127.0.0.1:38327_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node4":{
          "core":"collection1_shard1_replica_n2",
          "base_url":"http://127.0.0.1:35099/solr";,
          "node_name":"127.0.0.1:35099_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([31C2E1C55E476E63:B996DE1FF0BB039B]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:310)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:288)
        at 
org.apache.solr.cloud.TestCloudRecovery2.test(TestCloudRecovery2.java:106)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.security.AuditLoggerIntegrationTest.testAsyncQueueDrain

Error Message:
Expecting <2 callbacks in buffer, was 2

Stack Trace:
java.lang.AssertionError: Expecting <2 callbacks in buffer, was 2
        at 
__randomizedtesting.SeedInfo.seed([31C2E1C55E476E63:80DD1EE1EE84FDBC]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.assertTrue(Assert.java:41)
        at 
org.apache.solr.security.AuditLoggerIntegrationTest.testAsyncQueueDrain(AuditLoggerIntegrationTest.java:128)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 14651 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudRecovery2
   [junit4]   2> 3756632 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/init-core-data-001
   [junit4]   2> 3756649 WARN  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=34 numCloses=34
   [junit4]   2> 3756649 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3756650 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, 
clientAuth=0.0/0.0)
   [junit4]   2> 3756650 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001
   [junit4]   2> 3756651 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3756651 INFO  (ZkTestServer Run Thread) [    ] 
o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3756651 INFO  (ZkTestServer Run Thread) [    ] 
o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3756762 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:42122
   [junit4]   2> 3756762 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:42122
   [junit4]   2> 3756762 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1 42122
   [junit4]   2> 3756768 INFO  (zkConnectionManagerCallback-13455-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3756770 INFO  (zkConnectionManagerCallback-13457-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3756771 INFO  (zkConnectionManagerCallback-13459-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3756773 WARN  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3756773 WARN  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3756774 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3756774 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3756774 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3756774 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3756774 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 11.0.1+13-LTS
   [junit4]   2> 3756774 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 11.0.1+13-LTS
   [junit4]   2> 3756791 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3756791 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3756791 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3756792 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3756792 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3756792 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3756792 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@461775f1{/solr,null,AVAILABLE}
   [junit4]   2> 3756792 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@56541de5{/solr,null,AVAILABLE}
   [junit4]   2> 3756794 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@61996b64{HTTP/1.1,[http/1.1, 
h2c]}{127.0.0.1:35099}
   [junit4]   2> 3756794 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.s.Server Started @3757058ms
   [junit4]   2> 3756794 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=35099}
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@16f05dd1{HTTP/1.1,[http/1.1, 
h2c]}{127.0.0.1:38327}
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.s.Server Started @3757066ms
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=38327}
   [junit4]   2> 3756803 ERROR (jetty-launcher-13460-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
9.0.0
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3756803 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2019-04-15T20:34:00.294678Z
   [junit4]   2> 3756804 ERROR (jetty-launcher-13460-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3756804 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3756804 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
9.0.0
   [junit4]   2> 3756804 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3756804 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3756804 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2019-04-15T20:34:00.295352Z
   [junit4]   2> 3756805 INFO  (zkConnectionManagerCallback-13462-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3756805 INFO  (zkConnectionManagerCallback-13464-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3756806 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3756806 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3757413 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: 
WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3757427 WARN  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@1a25e283[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3757430 WARN  (jetty-launcher-13460-thread-1) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@59f36984[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3757451 INFO  (jetty-launcher-13460-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42122/solr
   [junit4]   2> 3757477 INFO  (zkConnectionManagerCallback-13472-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3757569 INFO  (zkConnectionManagerCallback-13474-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3757894 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:38327_solr
   [junit4]   2> 3757895 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.Overseer Overseer 
(id=74528159750946822-127.0.0.1:38327_solr-n_0000000000) starting
   [junit4]   2> 3758027 INFO  (zkConnectionManagerCallback-13481-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3758045 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster 
at 127.0.0.1:42122/solr ready
   [junit4]   2> 3758064 INFO  
(OverseerStateUpdate-74528159750946822-127.0.0.1:38327_solr-n_0000000000) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.Overseer Starting to work on the main 
queue : 127.0.0.1:38327_solr
   [junit4]   2> 3758064 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:38327_solr
   [junit4]   2> 3758065 INFO  
(OverseerStateUpdate-74528159750946822-127.0.0.1:38327_solr-n_0000000000) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3758110 INFO  (zkCallback-13480-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3758385 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system 
collection, keeping metrics history in memory.
   [junit4]   2> 3758605 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38327.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3758729 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38327.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3758729 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38327.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3758786 INFO  (jetty-launcher-13460-thread-1) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node1/.
   [junit4]   2> 3758891 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: 
WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3758911 WARN  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@3c815ab1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3758924 WARN  (jetty-launcher-13460-thread-2) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@6f4d3719[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3758925 INFO  (jetty-launcher-13460-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42122/solr
   [junit4]   2> 3758983 INFO  (zkConnectionManagerCallback-13488-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3759064 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 
0x108c6f767b80008, likely client has closed socket
   [junit4]   2> 3759125 INFO  (zkConnectionManagerCallback-13490-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3759564 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3759746 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.ZkController Publish 
node=127.0.0.1:35099_solr as DOWN
   [junit4]   2> 3759782 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating 
transient cache for 2147483647 transient cores
   [junit4]   2> 3759782 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:35099_solr
   [junit4]   2> 3759783 INFO  (zkCallback-13480-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3759783 INFO  (zkCallback-13473-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3759784 INFO  (zkCallback-13489-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3759928 INFO  (zkConnectionManagerCallback-13497-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3759933 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 3759934 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster 
at 127.0.0.1:42122/solr ready
   [junit4]   2> 3759935 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system 
collection, keeping metrics history in memory.
   [junit4]   2> 3760016 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35099.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3760068 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35099.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3760068 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35099.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3760069 INFO  (jetty-launcher-13460-thread-2) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node2/.
   [junit4]   2> 3760321 INFO  (zkConnectionManagerCallback-13503-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3760394 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3760395 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42122/solr ready
   [junit4]   2> 3760480 INFO  (qtp1280080955-31781) [n:127.0.0.1:35099_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 3760568 INFO  
(OverseerThreadFactory-7738-thread-1-processing-n:127.0.0.1:38327_solr) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.a.c.CreateCollectionCmd Create collection 
collection1
   [junit4]   2> 3760737 INFO  
(OverseerStateUpdate-74528159750946822-127.0.0.1:38327_solr-n_0000000000) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:38327/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3760800 INFO  
(OverseerStateUpdate-74528159750946822-127.0.0.1:38327_solr-n_0000000000) 
[n:127.0.0.1:38327_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:35099/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3761027 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr    
x:collection1_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create 
command 
qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3761088 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr    
x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create 
command 
qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3761088 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr    
x:collection1_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault 
Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3762153 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 3762281 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.IndexSchema [collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 3762300 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3762300 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n2' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 3762301 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35099.solr.core.collection1.shard1.replica_n2' (registry 
'solr.core.collection1.shard1.replica_n2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3762301 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrCore [[collection1_shard1_replica_n2] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node2/collection1_shard1_replica_n2],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/]
   [junit4]   2> 3762364 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 3762423 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 3762458 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3762458 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 3762458 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38327.solr.core.collection1.shard1.replica_n1' (registry 
'solr.core.collection1.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3762458 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node1/collection1_shard1_replica_n1],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node1/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 3762801 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 3762802 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3762804 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3762804 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3762838 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@2ab414f0[collection1_shard1_replica_n2] main]
   [junit4]   2> 3762840 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 3762841 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 3762854 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3762854 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1630913635386654720
   [junit4]   2> 3762894 INFO  
(searcherExecutor-7750-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[collection1_shard1_replica_n2] Registered new searcher 
Searcher@2ab414f0[collection1_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3762917 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/collection1/terms/shard1 to Terms{values={core_node4=0}, version=0}
   [junit4]   2> 3762917 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContextBase make sure parent is created 
/collections/collection1/leaders/shard1
   [junit4]   2> 3762949 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 3763021 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 3763021 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3763196 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3763196 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3763247 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@fd52df7[collection1_shard1_replica_n1] main]
   [junit4]   2> 3763250 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 3763250 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 3763251 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3763251 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1630913635802939392
   [junit4]   2> 3763270 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/collection1/terms/shard1 to Terms{values={core_node3=0, 
core_node4=0}, version=1}
   [junit4]   2> 3763270 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContextBase make sure parent is created 
/collections/collection1/leaders/shard1
   [junit4]   2> 3763288 INFO  
(searcherExecutor-7751-thread-1-processing-n:127.0.0.1:38327_solr 
x:collection1_shard1_replica_n1 c:collection1 s:shard1 r:core_node3) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SolrCore 
[collection1_shard1_replica_n1] Registered new searcher 
Searcher@fd52df7[collection1_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3763450 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3763450 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3763450 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:35099/solr/collection1_shard1_replica_n2/
   [junit4]   2> 3763451 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n2 
url=http://127.0.0.1:35099/solr START 
replicas=[http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/] 
nUpdates=100
   [junit4]   2> 3763452 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n2 
url=http://127.0.0.1:35099/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 3763512 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=50
   [junit4]   2> 3763512 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the 
next candidate
   [junit4]   2> 3763512 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 3763512 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node 
/collections/collection1/leaders/shard1/leader after winning as 
/collections/collection1/leader_elect/shard1/election/74528159750946825-core_node4-n_0000000000
   [junit4]   2> 3763514 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:35099/solr/collection1_shard1_replica_n2/ shard1
   [junit4]   2> 3763631 INFO  (zkCallback-13489-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3763636 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3763637 INFO  (qtp1280080955-31786) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=2610
   [junit4]   2> 3763758 INFO  (zkCallback-13489-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3763773 INFO  (zkCallback-13489-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3764500 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=3413
   [junit4]   2> 3764521 INFO  (qtp1280080955-31781) [n:127.0.0.1:35099_solr    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
45 seconds. Check all shard replicas
   [junit4]   2> 3764548 INFO  
(OverseerCollectionConfigSetProcessor-74528159750946822-127.0.0.1:38327_solr-n_0000000000)
 [n:127.0.0.1:38327_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 3764602 INFO  (zkCallback-13473-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3764602 INFO  (zkCallback-13473-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3764612 INFO  (zkCallback-13489-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3764616 INFO  (zkCallback-13489-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3764618 INFO  (qtp1280080955-31781) [n:127.0.0.1:35099_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=4137
   [junit4]   2> 3764618 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: 
collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 3764619 INFO  
(SUITE-TestCloudRecovery2-seed#[31C2E1C55E476E63]-worker) [    ] 
o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 3764893 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 3764895 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=1168100109
   [junit4]   2> 3764895 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:35099_solr
   [junit4]   2> 3764896 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.ZkController Publish this node as DOWN...
   [junit4]   2> 3764896 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.ZkController Publish node=127.0.0.1:35099_solr as DOWN
   [junit4]   2> 3764929 INFO  (coreCloseExecutor-7760-thread-1) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[collection1_shard1_replica_n2]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@325847e9
   [junit4]   2> 3764930 INFO  (coreCloseExecutor-7760-thread-1) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.core.collection1.shard1.replica_n2, tag=325847e9
   [junit4]   2> 3764930 INFO  (coreCloseExecutor-7760-thread-1) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@5d3ab0e4: rootName = 
solr_35099, domain = solr.core.collection1.shard1.replica_n2, service url = 
null, agent id = null] for registry solr.core.collection1.shard1.replica_n2 / 
com.codahale.metrics.MetricRegistry@415411eb
   [junit4]   2> 3764978 INFO  (coreCloseExecutor-7760-thread-1) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.collection1.shard1.leader, tag=325847e9
   [junit4]   2> 3765017 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, 
tag=null
   [junit4]   2> 3765017 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@6d34463b: rootName = 
solr_35099, domain = solr.node, service url = null, agent id = null] for 
registry solr.node / com.codahale.metrics.MetricRegistry@6516490d
   [junit4]   2> 3765035 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, 
tag=null
   [junit4]   2> 3765035 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@39c3d25d: rootName = 
solr_35099, domain = solr.jvm, service url = null, agent id = null] for 
registry solr.jvm / com.codahale.metrics.MetricRegistry@789b6352
   [junit4]   2> 3765054 INFO  (zkCallback-13489-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3765054 INFO  (zkCallback-13489-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3765066 INFO  (zkCallback-13473-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3765066 INFO  (zkCallback-13473-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3765072 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, 
tag=null
   [junit4]   2> 3765072 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@22dc9414: rootName = 
solr_35099, domain = solr.jetty, service url = null, agent id = null] for 
registry solr.jetty / com.codahale.metrics.MetricRegistry@3c495ece
   [junit4]   2> 3765072 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 3765090 INFO  (zkCallback-13480-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 3765090 INFO  (zkCallback-13502-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 3765090 INFO  (zkCallback-13473-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 3765104 INFO  (zkCallback-13473-thread-2) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be 
the new leader - try and sync
   [junit4]   2> 3765123 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.s.AbstractConnector Stopped ServerConnector@61996b64{HTTP/1.1,[http/1.1, 
h2c]}{127.0.0.1:0}
   [junit4]   2> 3765124 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@56541de5{/solr,null,UNAVAILABLE}
   [junit4]   2> 3765124 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] o.e.j.s.session 
node0 Stopped scavenging
   [junit4]   2> 3765220 INFO  (zkCallback-13473-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 3765220 INFO  (zkCallback-13473-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 3767606 INFO  (zkCallback-13473-thread-2) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/
   [junit4]   2> 3767606 INFO  (zkCallback-13473-thread-2) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync 
replicas to me
   [junit4]   2> 3767606 INFO  (zkCallback-13473-thread-2) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy 
http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/ has no replicas
   [junit4]   2> 3767606 INFO  (zkCallback-13473-thread-2) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase 
Creating leader registration node 
/collections/collection1/leaders/shard1/leader after winning as 
/collections/collection1/leader_elect/shard1/election/74528159750946822-core_node3-n_0000000001
   [junit4]   2> 3767608 INFO  (zkCallback-13473-thread-2) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the 
new leader: http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 3767627 INFO  (zkCallback-13473-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 3767627 INFO  (zkCallback-13473-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 3767632 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1630913640396750848,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 3767632 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@4057c6ac 
commitCommandVersion:1630913640396750848
   [junit4]   2> 3768040 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@2e340133[collection1_shard1_replica_n1] main]
   [junit4]   2> 3768042 INFO  
(searcherExecutor-7751-thread-1-processing-n:127.0.0.1:38327_solr 
x:collection1_shard1_replica_n1 c:collection1 s:shard1 r:core_node3) 
[n:127.0.0.1:38327_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SolrCore 
[collection1_shard1_replica_n1] Registered new searcher 
Searcher@2e340133[collection1_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(9.0.0):C100:[diagnostics={os=Linux,
 java.vendor=Oracle Corporation, java.version=11.0.1, 
java.vm.version=11.0.1+13-LTS, lucene.version=9.0.0, os.arch=amd64, 
java.runtime.version=11.0.1+13-LTS, source=flush, os.version=4.4.0-112-generic, 
timestamp=1555360451186}]:[attributes={Lucene50PostingsFormat.fstMode=AUTO, 
Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
   [junit4]   2> 3768042 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 3768043 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/collection1/terms/shard1 to Terms{values={core_node3=1, 
core_node4=0}, version=2}
   [junit4]   2> 3768043 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update params={commit=true&wt=javabin&version=2}{add=[0 
(1630913640391507968), 1 (1630913640393605120), 2 (1630913640393605121), 3 
(1630913640393605122), 4 (1630913640393605123), 5 (1630913640393605124), 6 
(1630913640393605125), 7 (1630913640393605126), 8 (1630913640393605127), 9 
(1630913640393605128), ... (100 adds)],commit=} 0 2904
   [junit4]   2> 3768043 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3768060 WARN  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3768061 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 35099 try number 1 ...
   [junit4]   2> 3768061 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] o.e.j.s.Server 
jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 11.0.1+13-LTS
   [junit4]   2> 3768063 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] o.e.j.s.session 
DefaultSessionIdManager workerName=node0
   [junit4]   2> 3768063 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] o.e.j.s.session 
No SessionScavenger set, using defaults
   [junit4]   2> 3768063 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] o.e.j.s.session 
node0 Scavenging every 660000ms
   [junit4]   2> 3768063 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@17ae2c8a{/solr,null,AVAILABLE}
   [junit4]   2> 3768064 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@6d5f1f63{HTTP/1.1,[http/1.1, 
h2c]}{127.0.0.1:35099}
   [junit4]   2> 3768064 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] o.e.j.s.Server 
Started @3768327ms
   [junit4]   2> 3768064 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=35099}
   [junit4]   2> 3768077 ERROR 
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3768077 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3768077 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
9.0.0
   [junit4]   2> 3768077 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3768077 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3768077 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2019-04-15T20:34:11.568527Z
   [junit4]   2> 3768079 INFO  (zkConnectionManagerCallback-13511-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3768080 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3768617 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: 
WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3768619 WARN  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@24faa20c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3768630 WARN  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@7501f7d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3768643 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42122/solr
   [junit4]   2> 3768645 INFO  (zkConnectionManagerCallback-13518-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3768647 INFO  (zkConnectionManagerCallback-13520-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3768667 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3768670 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.ZkController Publish node=127.0.0.1:35099_solr as DOWN
   [junit4]   2> 3768671 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 
2147483647 transient cores
   [junit4]   2> 3768680 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:35099_solr
   [junit4]   2> 3768681 INFO  (zkCallback-13502-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3768681 INFO  (zkCallback-13473-thread-2) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3768681 INFO  (zkCallback-13519-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3768696 INFO  (zkCallback-13480-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3768746 INFO  (zkConnectionManagerCallback-13527-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3768748 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3768757 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42122/solr 
ready
   [junit4]   2> 3768757 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics 
history in memory.
   [junit4]   2> 3768785 INFO  (zkCallback-13473-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3768789 INFO  (zkCallback-13473-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3768821 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35099.solr.node' 
(registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3768861 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35099.solr.jvm' 
(registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3768861 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35099.solr.jetty' 
(registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3768871 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node2/.
   [junit4]   2> 3768871 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [n:127.0.0.1:35099_solr  
  ] o.a.s.c.CorePropertiesLocator Cores are: [collection1_shard1_replica_n2]
   [junit4]   2> 3768889 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 3768926 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr    ] o.a.s.s.IndexSchema 
[collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 3768944 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with 
uniqueid field id
   [junit4]   2> 3768969 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 
9.0.0
   [junit4]   2> 3769005 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.s.IndexSchema 
[collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 3769007 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 
with uniqueid field id
   [junit4]   2> 3769007 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 
'collection1_shard1_replica_n2' using configuration from collection 
collection1, trusted=true
   [junit4]   2> 3769007 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35099.solr.core.collection1.shard1.replica_n2' (registry 
'solr.core.collection1.shard1.replica_n2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@fb034a0
   [junit4]   2> 3769008 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[[collection1_shard1_replica_n2] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node2/collection1_shard1_replica_n2],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery2_31C2E1C55E476E63-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/]
   [junit4]   2> 3769171 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog 
implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3769171 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: 
dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 
numVersionBuckets=65536
   [junit4]   2> 3769334 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3769334 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3769348 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening 
[Searcher@59f7589b[collection1_shard1_replica_n2] main]
   [junit4]   2> 3769349 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured 
ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 3769350 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at 
path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 3769350 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be 
reserved for 10000ms.
   [junit4]   2> 3769350 INFO  
(coreLoadExecutor-7770-thread-1-processing-n:127.0.0.1:35099_solr) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version 
in index or recent updates, using new clock 1630913642198204416
   [junit4]   2> 3769365 INFO  
(searcherExecutor-7771-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[collection1_shard1_replica_n2] Registered new searcher 
Searcher@59f7589b[collection1_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3769368 INFO  
(TEST-TestCloudRecovery2.test-seed#[31C2E1C55E476E63]) [    ] 
o.a.s.c.SolrCloudTestCase active replica count: 1 expected replica count: 2
   [junit4]   2> 3769368 INFO  (watches-13500-thread-1) [    ] 
o.a.s.c.SolrCloudTestCase active replica count: 1 expected replica count: 2
   [junit4]   2> 3769395 INFO  
(coreZkRegister-7762-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContextBase make 
sure parent is created /collections/collection1/leaders/shard1
   [junit4]   2> 3769397 INFO  
(coreZkRegister-7762-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.ZkController Core needs to 
recover:collection1_shard1_replica_n2
   [junit4]   2> 3769397 INFO  
(updateExecutor-13514-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 3769397 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Starting recovery 
process. recoveringAfterStartup=true
   [junit4]   2> 3769398 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy startupVersions is 
empty
   [junit4]   2> 3769399 INFO  (qtp262524014-31787) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr 
path=/admin/ping params={wt=javabin&version=2} hits=100 status=0 QTime=0
   [junit4]   2> 3769399 INFO  (qtp262524014-31787) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr 
path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3769399 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Begin buffering 
updates. core=[collection1_shard1_replica_n2]
   [junit4]   2> 3769399 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Starting to buffer updates. 
FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 3769399 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Publishing state of 
core [collection1_shard1_replica_n2] as recovering, leader is 
[http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/] and I am 
[http://127.0.0.1:35099/solr/collection1_shard1_replica_n2/]
   [junit4]   2> 3769400 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.ZkShardTerms Successful update of 
terms at /collections/collection1/terms/shard1 to 
Terms{values={core_node4_recovering=0, core_node3=1, core_node4=1}, version=3}
   [junit4]   2> 3769413 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Sending prep recovery 
command to [http://127.0.0.1:38327/solr]; [WaitForState: 
action=PREPRECOVERY&core=collection1_shard1_replica_n1&nodeName=127.0.0.1:35099_solr&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 3769414 INFO  (qtp262524014-31780) [n:127.0.0.1:38327_solr    
x:collection1_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp Going to wait for 
coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: 
true, onlyIfLeaderActive: true
   [junit4]   2> 3769414 INFO  (qtp262524014-31780) [n:127.0.0.1:38327_solr    
x:collection1_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp In 
WaitForState(recovering): collection=collection1, shard=shard1, 
thisCore=collection1_shard1_replica_n1, leaderDoesNotNeedRecovery=false, 
isLeader? true, live=true, checkLive=true, currentState=down, 
localState=active, nodeName=127.0.0.1:35099_solr, coreNodeName=core_node4, 
onlyIfActiveCheckResult=false, nodeProps: 
core_node4:{"core":"collection1_shard1_replica_n2","base_url":"http://127.0.0.1:35099/solr","node_name":"127.0.0.1:35099_solr","state":"down","type":"NRT","force_set_state":"false"}
   [junit4]   2> 3769544 INFO  (zkCallback-13502-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769544 INFO  (zkCallback-13519-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769547 INFO  (watches-13500-thread-1) [    ] 
o.a.s.c.SolrCloudTestCase active replica count: 1 expected replica count: 2
   [junit4]   2> 3769547 INFO  (zkCallback-13519-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769547 INFO  (zkCallback-13519-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769547 INFO  (zkCallback-13473-thread-2) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769547 INFO  (zkCallback-13519-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769563 INFO  (zkCallback-13473-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3769564 INFO  (watches-13475-thread-1) [    ] 
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, 
shard=shard1, thisCore=collection1_shard1_replica_n1, 
leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, 
currentState=recovering, localState=active, nodeName=127.0.0.1:35099_solr, 
coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: 
core_node4:{"core":"collection1_shard1_replica_n2","base_url":"http://127.0.0.1:35099/solr","node_name":"127.0.0.1:35099_solr","state":"recovering","type":"NRT","force_set_state":"false"}
   [junit4]   2> 3769564 INFO  (qtp262524014-31780) [n:127.0.0.1:38327_solr    
x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null 
path=/admin/cores 
params={nodeName=127.0.0.1:35099_solr&onlyIfLeaderActive=true&core=collection1_shard1_replica_n1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2}
 status=0 QTime=149
   [junit4]   2> 3770065 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Attempting to 
PeerSync from [http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/] - 
recoveringAfterStartup=[true]
   [junit4]   2> 3770066 WARN  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.PeerSyncWithLeader no frame of 
reference to tell if we've missed updates
   [junit4]   2> 3770066 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy PeerSync Recovery was 
not successful - trying replication.
   [junit4]   2> 3770066 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Starting Replication 
Recovery.
   [junit4]   2> 3770066 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.RecoveryStrategy Attempting to 
replicate from [http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/].
   [junit4]   2> 3770078 INFO  (qtp262524014-31784) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1630913642961567744,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 3770078 INFO  (qtp262524014-31784) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 3770079 INFO  (qtp262524014-31784) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 3770140 INFO  (qtp823637167-31880) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Ignoring commit while not ACTIVE - state: 
BUFFERING replay: false
   [junit4]   2> 3770140 INFO  (qtp823637167-31880) [n:127.0.0.1:35099_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=false&commit=true&softCommit=false&distrib.from=http://127.0.0.1:38327/solr/collection1_shard1_replica_n1/&commit_end_point=replicas&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 0
   [junit4]   2> 3770151 INFO  (qtp262524014-31784) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update 
params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
 0 72
   [junit4]   2> 3770167 INFO  (qtp262524014-31785) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 3770167 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 3770167 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Master's version: 
1555360451123
   [junit4]   2> 3770167 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 3770167 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 3770167 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Starting replication 
process
   [junit4]   2> 3770186 INFO  (qtp262524014-31782) [n:127.0.0.1:38327_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr 
path=/replication 
params={generation=2&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist}
 status=0 QTime=3
   [junit4]   2> 3770187 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Number of files in latest 
index in master: 21
   [junit4]   2> 3770236 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.u.DefaultSolrCoreState New IndexWriter 
is ready to be used.
   [junit4]   2> 3770237 INFO  
(recoveryExecutor-13516-thread-1-processing-n:127.0.0.1:35099_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:35099_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.h.IndexFetcher Starting download 
(fullCopy=false) to MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lu

[...truncated too long message...]

  2>    3       /solr/live_nodes
   [junit4]   2>        3       /solr/collections
   [junit4]   2> 
   [junit4]   2> 5687973 INFO  
(TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[31C2E1C55E476E63]) [ 
   ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44082
   [junit4]   2> 5687973 INFO  
(TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[31C2E1C55E476E63]) [ 
   ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44082
   [junit4]   2> 5687973 INFO  
(TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[31C2E1C55E476E63]) [ 
   ] o.a.s.SolrTestCaseJ4 ###Ending testAsyncQueueDrain
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=AuditLoggerIntegrationTest -Dtests.method=testAsyncQueueDrain 
-Dtests.seed=31C2E1C55E476E63 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.badapples=true -Dtests.locale=ar-MR -Dtests.timezone=America/Recife 
-Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 5.26s J1 | AuditLoggerIntegrationTest.testAsyncQueueDrain 
<<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Expecting <2 callbacks 
in buffer, was 2
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([31C2E1C55E476E63:80DD1EE1EE84FDBC]:0)
   [junit4]    >        at 
org.apache.solr.security.AuditLoggerIntegrationTest.testAsyncQueueDrain(AuditLoggerIntegrationTest.java:128)
   [junit4]    >        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    >        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    >        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    >        at 
java.base/java.lang.reflect.Method.invoke(Method.java:566)
   [junit4]    >        at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-master/solr/build/solr-core/test/J1/temp/solr.security.AuditLoggerIntegrationTest_31C2E1C55E476E63-001
   [junit4]   2> Apr 15, 2019 9:06:11 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene80): {}, 
docValues:{}, maxPointsInLeafNode=1605, maxMBSortInHeap=6.986873524542899, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@73a418eb),
 locale=ar-MR, timezone=America/Recife
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 11.0.1 
(64-bit)/cpus=4,threads=1,free=316528576,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestFuzzyAnalyzedSuggestions, LeaderVoteWaitTimeoutTest, 
TestNumericRangeQuery32, TaggerTest, HdfsBasicDistributedZkTest, 
ChangedSchemaMergeTest, ReplaceNodeTest, 
VMParamsZkACLAndCredentialsProvidersTest, CollectionTooManyReplicasTest, 
HdfsDirectoryTest, MultiSolrCloudTestCaseTest, 
HdfsTlogReplayBufferedWhileIndexingTest, TestFieldCacheSort, 
TestRandomFaceting, StressHdfsTest, TestAddFieldRealTimeGet, 
HdfsNNFailoverTest, TestCustomStream, TestComplexPhraseQParserPlugin, 
BasicAuthStandaloneTest, HLLSerializationTest, SolrInfoBeanTest, 
AnalyticsQueryTest, TestCloudPivotFacet, TestTolerantSearch, TestTlogReplica, 
MoveReplicaHDFSFailoverTest, DOMUtilTest, URLClassifyProcessorTest, 
TestWriterPerf, SpellingQueryConverterTest, QueryParsingTest, 
TestShortCircuitedRequests, ForceLeaderWithTlogReplicasTest, 
UpdateRequestProcessorFactoryTest, ManagedSchemaRoundRobinCloudTest, 
TermVectorComponentTest, ConcurrentCreateRoutedAliasTest, TestUtilizeNode, 
NodeLostTriggerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, 
TestGraphMLResponseWriter, TestRealTimeGet, 
DistribDocExpirationUpdateProcessorTest, ConfigureRecoveryStrategyTest, 
ExitableDirectoryReaderTest, DistributedMLTComponentTest, 
SearchRateTriggerIntegrationTest, SignatureUpdateProcessorFactoryTest, 
FieldMutatingUpdateProcessorTest, TestConfigSetImmutable, 
TestStressCloudBlindAtomicUpdates, TestSimComputePlanAction, 
TestStressRecovery, CloudExitableDirectoryReaderTest, 
OverseerCollectionConfigSetProcessorTest, DirectSolrConnectionTest, 
ReindexCollectionTest, BitVectorTest, TestLogWatcher, 
TestPayloadCheckQParserPlugin, SolrJmxReporterCloudTest, 
TestNestedUpdateProcessor, IndexSizeTriggerTest, TestSolr4Spatial, 
TestValueSourceCache, SliceStateTest, TestFastOutputStream, 
ExecutePlanActionTest, TestFiltering, LargeFieldTest, PeerSyncTest, 
TestSolrCloudWithHadoopAuthPlugin, AdminHandlersProxyTest, 
TestSlowCompositeReaderWrapper, TokenizerChainTest, TestReplicaProperties, 
TestLockTree, SystemInfoHandlerTest, ResourceLoaderTest, 
TestPartialUpdateDeduplication, TestCollationField, SpellCheckCollatorTest, 
TestReplicationHandlerDiskOverFlow, TestRequestForwarding, 
DistributedVersionInfoTest, TestQueryTypes, SchemaApiFailureTest, 
ResponseBuilderTest, TestAuthorizationFramework, TriggerIntegrationTest, 
ZkSolrClientTest, BasicFunctionalityTest, ZkStateWriterTest, 
TestCursorMarkWithoutUniqueKey, RegexBoostProcessorTest, TestFastLRUCache, 
TestRemoteStreaming, MetricsHistoryHandlerTest, 
LeaderFailoverAfterPartitionTest, TestOrdValues, ZkFailoverTest, 
BlockCacheTest, TestMaxScoreQueryParser, PhrasesIdentificationComponentTest, 
DistributedFacetPivotSmallAdvancedTest, HdfsRestartWhileUpdatingTest, 
HighlighterTest, DistributedFacetSimpleRefinementLongTailTest, 
TestRetrieveFieldsOptimizer, SolrCoreTest, TestSolrCloudWithDelegationTokens, 
DateFieldTest, ScheduledMaintenanceTriggerTest, 
TestInPlaceUpdatesRequiredField, TestSystemIdResolver, 
TestRandomCollapseQParserPlugin, SuggesterFSTTest, 
PeerSyncWithLeaderAndIndexFingerprintCachingTest, BasicDistributedZk2Test, 
BlockJoinFacetSimpleTest, TestCollectionsAPIViaSolrCloudCluster, TestCSVLoader, 
OverseerStatusTest, ChaosMonkeyNothingIsSafeTest, MetricUtilsTest, TestSort, 
CreateCollectionCleanupTest, NodeMutatorTest, TestJsonRequest, 
TestChildDocTransformerHierarchy, TlogReplayBufferedWhileIndexingTest, 
TestHdfsBackupRestoreCore, TestOmitPositions, TestConfigsApi, 
ProtectedTermFilterFactoryTest, SolrXmlInZkTest, DistributedTermsComponentTest, 
TestCSVResponseWriter, ConvertedLegacyTest, CursorPagingTest, 
DisMaxRequestHandlerTest, DistributedIntervalFacetingTest, EchoParamsTest, 
MinimalSchemaTest, OutputWriterTest, SolrTestCaseJ4Test, 
TestDistributedGrouping, TestDistributedMissingSort, TestDistributedSearch, 
TestDeprecatedFilters, TestLuceneMatchVersion, 
TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, 
TestJettySolrRunner, ConnectionReuseTest, AliasIntegrationTest, 
CollectionStateFormat2Test, DeleteInactiveReplicaTest, DeleteShardTest, 
DeleteStatusTest, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, 
HealthCheckHandlerTest, HttpPartitionOnCommitTest, LegacyCloudClusterPropTest, 
MetricsHistoryIntegrationTest, MetricsHistoryWithAuthIntegrationTest, 
MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, 
OverseerTaskQueueTest, PeerSyncReplicationTest, SystemCollectionCompatTest, 
TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudRecovery, 
TestClusterProperties, TestConfigSetsAPI, TestConfigSetsAPIZkFailure, 
TestCryptoKeys, TestPullReplicaErrorHandling, TestSSLRandomization, 
TestSizeLimitedDistributedMap, TestSolrCloudWithKerberosAlt, 
TestTolerantUpdateProcessorCloud, ZkCLITest, SimpleCollectionCreateDeleteTest, 
TestHdfsCloudBackupRestore, TestRequestStatusCollectionAPI, 
AutoAddReplicasIntegrationTest, SystemLogListenerTest, 
TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest, 
TestSimLargeCluster, TestQuerySenderListener, TestQuerySenderNoQuery, 
TestSolrConfigHandler, DocumentAnalysisRequestHandlerTest, 
MoreLikeThisHandlerTest, TestReqParamsAPI, TestRestoreCore, 
TestSolrConfigHandlerConcurrent, TestSystemCollAutoCreate, 
V2ApiIntegrationTest, V2StandaloneTest, XmlUpdateRequestHandlerTest, 
AutoscalingHistoryHandlerTest, TestApiFramework, ZookeeperStatusHandlerTest, 
BadComponentTest, DistributedDebugComponentTest, 
DistributedFacetExistsSmallTest, DistributedQueryComponentOptimizationTest, 
ReplicaListTransformerTest, ResponseLogComponentTest, 
ShufflingReplicaListTransformerTest, StatsComponentTest, TestExportWriter, 
JavabinLoaderTest, EmbeddedSolrNoSerializeTest, RandomizedTaggerTest, 
TaggingAttributeTest, FastVectorHighlighterTest, WrapperMergePolicyFactoryTest, 
TestLegacyField, TestLegacyFieldReuse, TestLegacyNumericUtils, 
MetricsConfigTest, SolrCoreMetricManagerTest, SolrMetricManagerTest, 
SolrMetricReporterTest, SolrMetricsIntegrationTest, SolrSlf4jReporterTest, 
SolrRrdBackendFactoryTest, RegexBytesRefFilterTest, SimpleFacetsTest, 
SmileWriterTest, TestPushWriter, TestRawResponseWriter, 
TestSubQueryTransformerCrossCore, TestManagedResourceStorage, TestRestManager, 
TestCopyFieldCollectionResource, TestDynamicFieldCollectionResource, 
TestFieldCollectionResource, TestFieldTypeCollectionResource, 
TestSchemaNameResource, TestSchemaVersionResource, 
TestManagedStopFilterFactory, BadCopyFieldTest, BadIndexSchemaTest, 
DocValuesMultiTest, DocValuesTest, TestHalfAndHalfDocValues, TestManagedSchema, 
TestUseDocValuesAsStored2, UUIDFieldTest, CurrencyRangeFacetCloudTest, 
RankQueryTest, ReturnFieldsTest, SignificantTermsQParserPluginTest, 
TestCollapseQParserPlugin, TestDocSet, TestHashQParserPlugin, TestInitQParser, 
TestLFUCache, TestRecoveryHdfs, TestJsonFacetsStatsParsing, 
TestJsonFacetsWithNestedObjects, SortByFunctionTest, TestFunctionQuery, 
DistanceFunctionTest, BJQParserTest, TestCloudNestedDocsSort, 
TestNestedDocsSort, SimpleMLTQParserTest, TestBM25SimilarityFactory, 
TestDFISimilarityFactory, TestDFRSimilarityFactory, TestIBSimilarityFactory, 
TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, 
TestNonDefinedSimilarityFactory, TestPerFieldSimilarity, TestDefaultStatsCache, 
TestDistribIDF, TestExactStatsCache, AuditLoggerIntegrationTest]
   [junit4] Completed [773/854 (2!)] on J1 in 24.97s, 7 tests, 1 failure <<< 
FAILURES!

[...truncated 43446 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to