Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.2/7/
4 tests failed.
FAILED:
junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
6 threads leaked from SUITE scope at
org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest: 1)
Thread[id=21907, name=zkCallback-281-thread-4, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 2) Thread[id=7242,
name=zkConnectionManagerCallback-282-thread-1, state=WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 3) Thread[id=21906,
name=zkCallback-281-thread-3, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 4) Thread[id=7241,
name=StoppableIndexingThread-EventThread, state=WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=21849, name=zkCallback-281-thread-2, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 6) Thread[id=7240,
name=StoppableIndexingThread-SendThread(127.0.0.1:37307), state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
java.lang.Thread.sleep(Native Method) at
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE
scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:
1) Thread[id=21907, name=zkCallback-281-thread-4, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=7242, name=zkConnectionManagerCallback-282-thread-1,
state=WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=21906, name=zkCallback-281-thread-3, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=7241, name=StoppableIndexingThread-EventThread, state=WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=21849, name=zkCallback-281-thread-2, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=7240, name=StoppableIndexingThread-SendThread(127.0.0.1:37307),
state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([9910A7879E9A95D8]:0)
FAILED:
junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1)
Thread[id=21907, name=zkCallback-281-thread-4, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 2) Thread[id=7242,
name=zkConnectionManagerCallback-282-thread-1, state=WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 3) Thread[id=21906,
name=zkCallback-281-thread-3, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 4) Thread[id=21849,
name=zkCallback-281-thread-2, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
sun.misc.Unsafe.park(Native Method) at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) 5) Thread[id=7240,
name=StoppableIndexingThread-SendThread(127.0.0.1:37307), state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest] at
java.lang.Thread.sleep(Native Method) at
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie
threads that couldn't be terminated:
1) Thread[id=21907, name=zkCallback-281-thread-4, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=7242, name=zkConnectionManagerCallback-282-thread-1,
state=WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=21906, name=zkCallback-281-thread-3, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=21849, name=zkCallback-281-thread-2, state=TIMED_WAITING,
group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=7240, name=StoppableIndexingThread-SendThread(127.0.0.1:37307),
state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
at __randomizedtesting.SeedInfo.seed([9910A7879E9A95D8]:0)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Expected numSlices=5 numReplicas=1 but found
DocCollection(solrj_collection4//collections/solrj_collection4/state.json/26)={
"pullReplicas":"0", "replicationFactor":"1", "shards":{ "shard1":{
"range":"80000000-b332ffff", "state":"active",
"replicas":{"core_node4":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node4/data/",
"base_url":"http://127.0.0.1:44676";,
"node_name":"127.0.0.1:44676_", "type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node4/data/tlog",
"core":"solrj_collection4_shard1_replica_n1",
"shared_storage":"true", "state":"active",
"leader":"true"}}}, "shard2":{ "range":"b3330000-e665ffff",
"state":"active", "replicas":{"core_node6":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node6/data/",
"base_url":"http://127.0.0.1:39324";,
"node_name":"127.0.0.1:39324_", "type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node6/data/tlog",
"core":"solrj_collection4_shard2_replica_n2",
"shared_storage":"true", "state":"active",
"leader":"true"}}}, "shard3":{ "range":"e6660000-1998ffff",
"state":"active", "replicas":{"core_node8":{
"core":"solrj_collection4_shard3_replica_n3",
"base_url":"http://127.0.0.1:39324";, "node_name":"127.0.0.1:39324_",
"state":"down", "type":"NRT"}}}, "shard4":{
"range":"19990000-4ccbffff", "state":"active",
"replicas":{"core_node9":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node9/data/",
"base_url":"http://127.0.0.1:39324";,
"node_name":"127.0.0.1:39324_", "type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
"core":"solrj_collection4_shard4_replica_n5",
"shared_storage":"true", "state":"active",
"leader":"true"}}}, "shard5":{ "range":"4ccc0000-7fffffff",
"state":"active", "replicas":{"core_node10":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node10/data/",
"base_url":"http://127.0.0.1:35783";,
"node_name":"127.0.0.1:35783_", "type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node10/data/tlog",
"core":"solrj_collection4_shard5_replica_n7",
"shared_storage":"true", "state":"active",
"leader":"true"}}}}, "router":{ "field":"text",
"name":"compositeId"}, "maxShardsPerNode":"5", "autoAddReplicas":"true",
"nrtReplicas":"1", "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:35783_,
127.0.0.1:37030_, 127.0.0.1:39324_, 127.0.0.1:44676_, 127.0.0.1:45543_]
Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found
DocCollection(solrj_collection4//collections/solrj_collection4/state.json/26)={
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{
"shard1":{
"range":"80000000-b332ffff",
"state":"active",
"replicas":{"core_node4":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node4/data/",
"base_url":"http://127.0.0.1:44676";,
"node_name":"127.0.0.1:44676_",
"type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node4/data/tlog",
"core":"solrj_collection4_shard1_replica_n1",
"shared_storage":"true",
"state":"active",
"leader":"true"}}},
"shard2":{
"range":"b3330000-e665ffff",
"state":"active",
"replicas":{"core_node6":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node6/data/",
"base_url":"http://127.0.0.1:39324";,
"node_name":"127.0.0.1:39324_",
"type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node6/data/tlog",
"core":"solrj_collection4_shard2_replica_n2",
"shared_storage":"true",
"state":"active",
"leader":"true"}}},
"shard3":{
"range":"e6660000-1998ffff",
"state":"active",
"replicas":{"core_node8":{
"core":"solrj_collection4_shard3_replica_n3",
"base_url":"http://127.0.0.1:39324";,
"node_name":"127.0.0.1:39324_",
"state":"down",
"type":"NRT"}}},
"shard4":{
"range":"19990000-4ccbffff",
"state":"active",
"replicas":{"core_node9":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node9/data/",
"base_url":"http://127.0.0.1:39324";,
"node_name":"127.0.0.1:39324_",
"type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
"core":"solrj_collection4_shard4_replica_n5",
"shared_storage":"true",
"state":"active",
"leader":"true"}}},
"shard5":{
"range":"4ccc0000-7fffffff",
"state":"active",
"replicas":{"core_node10":{
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node10/data/",
"base_url":"http://127.0.0.1:35783";,
"node_name":"127.0.0.1:35783_",
"type":"NRT",
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node10/data/tlog",
"core":"solrj_collection4_shard5_replica_n7",
"shared_storage":"true",
"state":"active",
"leader":"true"}}}},
"router":{
"field":"text",
"name":"compositeId"},
"maxShardsPerNode":"5",
"autoAddReplicas":"true",
"nrtReplicas":"1",
"tlogReplicas":"0"} with /live_nodes: [127.0.0.1:35783_, 127.0.0.1:37030_,
127.0.0.1:39324_, 127.0.0.1:44676_, 127.0.0.1:45543_]
at
__randomizedtesting.SeedInfo.seed([9910A7879E9A95D8:1144985D3066F820]:0)
at org.junit.Assert.fail(Assert.java:93)
at
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
at
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:258)
at
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth
investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties
were stopped - this is worth investigating!
at
__randomizedtesting.SeedInfo.seed([9910A7879E9A95D8:1144985D3066F820]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at
org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12782 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/init-core-data-001
[junit4] 2> 711758 WARN
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 711829 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 711831 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via:
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 711831 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 711831 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 714665 WARN
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your
platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 717692 WARN
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 718522 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via
org.mortbay.log.Slf4jLog
[junit4] 2> 718625 WARN
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 720331 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log jetty-6.1.26
[junit4] 2> 720552 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Extract
jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs
to ./temp/Jetty_lucene2.us.west_apache_org_44700_hdfs____t3id7l/webapp
[junit4] 2> 723338 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Started
HttpServer2$selectchannelconnectorwithsafestar...@lucene2-us-west.apache.org:44700
[junit4] 2> 727690 WARN
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 727707 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log jetty-6.1.26
[junit4] 2> 727768 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Extract
jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode
to ./temp/Jetty_localhost_35157_datanode____.v9lj8o/webapp
[junit4] 2> 728728 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Started
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35157
[junit4] 2> 731555 WARN
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 731673 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log jetty-6.1.26
[junit4] 2> 731919 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Extract
jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode
to ./temp/Jetty_localhost_42686_datanode____.l6qlyb/webapp
[junit4] 2> 734090 INFO
(SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Started
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42686
[junit4] 2> 739532 ERROR (DataNode:
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data3/,
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data4/]]
heartbeating to lucene2-us-west.apache.org/127.0.0.1:45565) [ ]
o.a.h.h.s.d.DirectoryScanner
dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1
ms/sec. Assuming default value of 1000
[junit4] 2> 739532 ERROR (DataNode:
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data1/,
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data2/]]
heartbeating to lucene2-us-west.apache.org/127.0.0.1:45565) [ ]
o.a.h.h.s.d.DirectoryScanner
dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1
ms/sec. Assuming default value of 1000
[junit4] 2> 742261 INFO (Block report processor) [ ] BlockStateChange
BLOCK* processReport 0x99128259307f: from storage
DS-ba42bff3-d699-4ac2-82df-0770f674b18c node
DatanodeRegistration(127.0.0.1:35176,
datanodeUuid=a8d89727-bc24-4f32-9887-88a5b58818ab, infoPort=41079,
infoSecurePort=0, ipcPort=40686,
storageInfo=lv=-56;cid=testClusterID;nsid=80848523;c=0), blocks: 0,
hasStaleStorage: true, processing time: 96 msecs
[junit4] 2> 742267 INFO (Block report processor) [ ] BlockStateChange
BLOCK* processReport 0x99128259307f: from storage
DS-524ad0d4-63d5-4c10-85c9-4ca2cb6a8a90 node
DatanodeRegistration(127.0.0.1:35176,
datanodeUuid=a8d89727-bc24-4f32-9887-88a5b58818ab, infoPort=41079,
infoSecurePort=0, ipcPort=40686,
storageInfo=lv=-56;cid=testClusterID;nsid=80848523;c=0), blocks: 0,
hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 742332 INFO (Block report processor) [ ] BlockStateChange
BLOCK* processReport 0x9912836daccc: from storage
DS-e93660bb-91fc-4726-8271-131628bad3aa node
DatanodeRegistration(127.0.0.1:38952,
datanodeUuid=d9fc1e54-c960-45a2-bbcb-83a34b0edaf0, infoPort=41281,
infoSecurePort=0, ipcPort=44198,
storageInfo=lv=-56;cid=testClusterID;nsid=80848523;c=0), blocks: 0,
hasStaleStorage: true, processing time: 49 msecs
[junit4] 2> 742332 INFO (Block report processor) [ ] BlockStateChange
BLOCK* processReport 0x9912836daccc: from storage
DS-49925226-0105-45d2-9c56-1972abb141a1 node
DatanodeRegistration(127.0.0.1:38952,
datanodeUuid=d9fc1e54-c960-45a2-bbcb-83a34b0edaf0, infoPort=41281,
infoSecurePort=0, ipcPort=44198,
storageInfo=lv=-56;cid=testClusterID;nsid=80848523;c=0), blocks: 0,
hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 745250 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 745287 INFO (Thread-514) [ ] o.a.s.c.ZkTestServer client
port:0.0.0.0/0.0.0.0:0
[junit4] 2> 745287 INFO (Thread-514) [ ] o.a.s.c.ZkTestServer Starting
server
[junit4] 2> 745336 ERROR (Thread-514) [ ] o.a.z.s.ZooKeeperServer
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action
on ERROR or SHUTDOWN server state changes
[junit4] 2> 745375 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.ZkTestServer start zk server on port:37307
[junit4] 2> 745429 INFO (zkConnectionManagerCallback-192-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 745510 INFO (zkConnectionManagerCallback-194-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 745566 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
to /configs/conf1/solrconfig.xml
[junit4] 2> 745568 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml
to /configs/conf1/schema.xml
[junit4] 2> 745570 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 745583 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
to /configs/conf1/stopwords.txt
[junit4] 2> 745596 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt
to /configs/conf1/protwords.txt
[junit4] 2> 745597 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml
to /configs/conf1/currency.xml
[junit4] 2> 745598 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
to /configs/conf1/enumsConfig.xml
[junit4] 2> 745628 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
to /configs/conf1/open-exchange-rates.json
[junit4] 2> 745629 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 745631 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
to /configs/conf1/old_synonyms.txt
[junit4] 2> 745632 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
to /configs/conf1/synonyms.txt
[junit4] 2> 745634 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly
asked otherwise
[junit4] 2> 746170 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 746184 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.h.ContextHandler Started
o.e.j.s.ServletContextHandler@1c39b0ec{/,null,AVAILABLE}
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.AbstractConnector Started
ServerConnector@76d99ab8{HTTP/1.1,[http/1.1]}{127.0.0.1:42720}
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.Server Started @756989ms
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.s.e.JettySolrRunner Jetty properties:
{solr.data.dir=hdfs://lucene2-us-west.apache.org:45565/hdfs__lucene2-us-west.apache.org_45565__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.2_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001_tempDir-002_control_data,
hostContext=/, hostPort=42720,
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/control-001/cores}
[junit4] 2> 746216 ERROR
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be
missing or incomplete.
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version
7.2.1
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 746216 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
2018-01-11T23:39:36.879Z
[junit4] 2> 746249 INFO (zkConnectionManagerCallback-196-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 746250 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in
ZooKeeper)
[junit4] 2> 746251 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig Loading container configuration from
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/control-001/solr.xml
[junit4] 2> 746255 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay
is ignored
[junit4] 2> 746255 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig Configuration parameter
autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 746273 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig MBean server found:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0, but no JMX reporters were
configured - adding default JMX reporter.
[junit4] 2> 746288 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37307/solr
[junit4] 2> 746330 INFO (zkConnectionManagerCallback-200-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 746395 INFO
(zkConnectionManagerCallback-202-thread-1-processing-n:127.0.0.1:42720_)
[n:127.0.0.1:42720_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 746665 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 746678 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.c.OverseerElectionContext I am going to be the
leader 127.0.0.1:42720_
[junit4] 2> 746679 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.c.Overseer Overseer
(id=99333831128907780-127.0.0.1:42720_-n_0000000000) starting
[junit4] 2> 746758 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.c.ZkController Register node as live in
ZooKeeper:/live_nodes/127.0.0.1:42720_
[junit4] 2> 746790 INFO
(zkCallback-201-thread-1-processing-n:127.0.0.1:42720_) [n:127.0.0.1:42720_
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 747065 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.node' (registry 'solr.node') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 747121 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.jvm' (registry 'solr.jvm') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 747122 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.jetty' (registry 'solr.jetty') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 747123 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:42720_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
underneath
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/control-001/cores
[junit4] 2> 747231 INFO (zkConnectionManagerCallback-208-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 747232 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 747233 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37307/solr ready
[junit4] 2> 747274 INFO (qtp56662695-1323) [n:127.0.0.1:42720_ ]
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params
replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42720_&wt=javabin&version=2
and sendToOCPQueue=true
[junit4] 2> 747277 INFO
(OverseerThreadFactory-337-thread-1-processing-n:127.0.0.1:42720_)
[n:127.0.0.1:42720_ ] o.a.s.c.CreateCollectionCmd Create collection
control_collection
[junit4] 2> 747449 INFO (qtp56662695-1325) [n:127.0.0.1:42720_ ]
o.a.s.h.a.CoreAdminOperation core create command
qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 747450 INFO (qtp56662695-1325) [n:127.0.0.1:42720_ ]
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4
transient cores
[junit4] 2> 747562 INFO
(zkCallback-201-thread-1-processing-n:127.0.0.1:42720_) [n:127.0.0.1:42720_
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/control_collection/state.json] for collection
[control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 748525 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.1
[junit4] 2> 748651 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 749362 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 749489 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1'
using configuration from collection control_collection, trusted=true
[junit4] 2> 749490 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.core.control_collection.shard1.replica_n1' (registry
'solr.core.control_collection.shard1.replica_n1') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 749636 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory
solr.hdfs.home=hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home
[junit4] 2> 749636 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 749636 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 749637 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore
at
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/control-001/cores/control_collection_shard1_replica_n1],
dataDir=[hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/control_collection/core_node2/data/]
[junit4] 2> 749670 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory creating directory factory for path
hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
[junit4] 2> 749782 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct
memory allocation set to [true]
[junit4] 2> 749782 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of
[8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 749783 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 752265 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 752282 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory creating directory factory for path
hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/control_collection/core_node2/data
[junit4] 2> 752556 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory creating directory factory for path
hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/control_collection/core_node2/data/index
[junit4] 2> 752575 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct
memory allocation set to [true]
[junit4] 2> 752575 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of
[8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 752576 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 752629 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 752630 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy:
minMergeSize=1000, mergeFactor=16, maxMergeSize=9223372036854775807,
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false,
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.8471231011159144]
[junit4] 2> 753900 INFO (Block report processor) [ ] BlockStateChange
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38952 is added to
blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1,
replicas=[ReplicaUC[[DISK]DS-49925226-0105-45d2-9c56-1972abb141a1:NORMAL:127.0.0.1:38952|RBW],
ReplicaUC[[DISK]DS-524ad0d4-63d5-4c10-85c9-4ca2cb6a8a90:NORMAL:127.0.0.1:35176|RBW]]}
size 69
[junit4] 2> 753901 INFO (Block report processor) [ ] BlockStateChange
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:35176 is added to
blk_1073741825_1001 size 69
[junit4] 2> 754441 WARN (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type =
requestHandler,name = /dump,class = DumpRequestHandler,attributes =
{initParams=a, name=/dump, class=DumpRequestHandler},args =
{defaults={a=A,b=B}}}
[junit4] 2> 754653 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.UpdateHandler Using UpdateLog implementation:
org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 754653 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 754653 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 754709 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 754709 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 754758 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648,
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false,
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.0]
[junit4] 2> 754924 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.s.SolrIndexSearcher Opening
[Searcher@45e1a2a0[control_collection_shard1_replica_n1] main]
[junit4] 2> 754937 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase:
/configs/conf1
[junit4] 2> 754938 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using
ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 754940 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 754944 INFO
(searcherExecutor-340-thread-1-processing-n:127.0.0.1:42720_
x:control_collection_shard1_replica_n1 s:shard1 c:control_collection)
[n:127.0.0.1:42720_ c:control_collection s:shard1
x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore
[control_collection_shard1_replica_n1] Registered new searcher
Searcher@45e1a2a0[control_collection_shard1_replica_n1]
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 754960 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using
new clock 1589341308171845632
[junit4] 2> 754977 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 754977 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 754978 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.SyncStrategy Sync replicas to
http://127.0.0.1:42720/control_collection_shard1_replica_n1/
[junit4] 2> 754978 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 754978 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.SyncStrategy
http://127.0.0.1:42720/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 754978 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in
election, clear LIR
[junit4] 2> 754989 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.ShardLeaderElectionContext I am the new leader:
http://127.0.0.1:42720/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 754990 INFO
(zkCallback-201-thread-1-processing-n:127.0.0.1:42720_) [n:127.0.0.1:42720_
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/control_collection/state.json] for collection
[control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 755040 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 755042 INFO (qtp56662695-1325) [n:127.0.0.1:42720_
c:control_collection s:shard1 x:control_collection_shard1_replica_n1]
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores
params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT}
status=0 QTime=7592
[junit4] 2> 755055 INFO (qtp56662695-1323) [n:127.0.0.1:42720_ ]
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most
30 seconds. Check all shard replicas
[junit4] 2> 755143 INFO
(zkCallback-201-thread-2-processing-n:127.0.0.1:42720_) [n:127.0.0.1:42720_
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/control_collection/state.json] for collection
[control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 755342 INFO
(OverseerCollectionConfigSetProcessor-99333831128907780-127.0.0.1:42720_-n_0000000000)
[n:127.0.0.1:42720_ ] o.a.s.c.OverseerTaskQueue Response ZK path:
/overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may
have disconnected from ZooKeeper
[junit4] 2> 756055 INFO (qtp56662695-1323) [n:127.0.0.1:42720_ ]
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections
params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42720_&wt=javabin&version=2}
status=0 QTime=8781
[junit4] 2> 756077 INFO (zkConnectionManagerCallback-212-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 756078 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 756079 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37307/solr ready
[junit4] 2> 756079 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection
loss:false
[junit4] 2> 756080 INFO (qtp56662695-1324) [n:127.0.0.1:42720_ ]
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params
replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2
and sendToOCPQueue=true
[junit4] 2> 756104 INFO
(OverseerThreadFactory-337-thread-2-processing-n:127.0.0.1:42720_)
[n:127.0.0.1:42720_ ] o.a.s.c.CreateCollectionCmd Create collection
collection1
[junit4] 2> 756105 WARN
(OverseerThreadFactory-337-thread-2-processing-n:127.0.0.1:42720_)
[n:127.0.0.1:42720_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a
collection (collection1) without cores.
[junit4] 2> 756356 INFO (qtp56662695-1324) [n:127.0.0.1:42720_ ]
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most
30 seconds. Check all shard replicas
[junit4] 2> 756356 INFO (qtp56662695-1324) [n:127.0.0.1:42720_ ]
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections
params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2}
status=0 QTime=275
[junit4] 2> 756666 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/shard-1-001
of type NRT
[junit4] 2> 756667 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 756675 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.h.ContextHandler Started
o.e.j.s.ServletContextHandler@4a8a743e{/,null,AVAILABLE}
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.AbstractConnector Started
ServerConnector@7047e76a{HTTP/1.1,[http/1.1]}{127.0.0.1:44639}
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.e.j.s.Server Started @767449ms
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.s.e.JettySolrRunner Jetty properties:
{solr.data.dir=hdfs://lucene2-us-west.apache.org:45565/hdfs__lucene2-us-west.apache.org_45565__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.2_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001_tempDir-002_jetty1,
solrconfig=solrconfig.xml, hostContext=/, hostPort=44639,
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/shard-1-001/cores}
[junit4] 2> 756676 ERROR
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be
missing or incomplete.
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version
7.2.1
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 756676 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
2018-01-11T23:39:47.339Z
[junit4] 2> 756694 INFO (zkConnectionManagerCallback-214-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 756757 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in
ZooKeeper)
[junit4] 2> 756757 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig Loading container configuration from
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/shard-1-001/solr.xml
[junit4] 2> 756774 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay
is ignored
[junit4] 2> 756774 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig Configuration parameter
autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 756775 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.SolrXmlConfig MBean server found:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0, but no JMX reporters were
configured - adding default JMX reporter.
[junit4] 2> 756793 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37307/solr
[junit4] 2> 756838 INFO (zkConnectionManagerCallback-218-thread-1) [ ]
o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 756931 INFO
(zkConnectionManagerCallback-220-thread-1-processing-n:127.0.0.1:44639_)
[n:127.0.0.1:44639_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 756954 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.c.c.ZkStateReader Updated live nodes from
ZooKeeper... (0) -> (1)
[junit4] 2> 756973 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 756992 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.c.ZkController Register node as live in
ZooKeeper:/live_nodes/127.0.0.1:44639_
[junit4] 2> 756993 INFO
(zkCallback-201-thread-2-processing-n:127.0.0.1:42720_) [n:127.0.0.1:42720_
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 757081 INFO (zkCallback-211-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 757199 INFO
(zkCallback-219-thread-1-processing-n:127.0.0.1:44639_) [n:127.0.0.1:44639_
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 757344 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.node' (registry 'solr.node') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 757377 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.jvm' (registry 'solr.jvm') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 757377 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.jetty' (registry 'solr.jetty') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 757379 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8])
[n:127.0.0.1:44639_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
underneath
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/shard-1-001/cores
[junit4] 2> 757467 INFO (qtp56662695-1320) [n:127.0.0.1:42720_ ]
o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params
node=127.0.0.1:44639_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2
and sendToOCPQueue=true
[junit4] 2> 757469 INFO
(OverseerCollectionConfigSetProcessor-99333831128907780-127.0.0.1:42720_-n_0000000000)
[n:127.0.0.1:42720_ ] o.a.s.c.OverseerTaskQueue Response ZK path:
/overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may
have disconnected from ZooKeeper
[junit4] 2> 757479 INFO
(OverseerThreadFactory-337-thread-3-processing-n:127.0.0.1:42720_)
[n:127.0.0.1:42720_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:44639_
for creating new replica
[junit4] 2> 757561 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_ ]
o.a.s.h.a.CoreAdminOperation core create command
qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 757562 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_ ]
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4
transient cores
[junit4] 2> 757698 INFO
(zkCallback-219-thread-1-processing-n:127.0.0.1:44639_) [n:127.0.0.1:44639_
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [2])
[junit4] 2> 758820 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrConfig
Using Lucene MatchVersion: 7.2.1
[junit4] 2> 759042 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema
[collection1_shard2_replica_n61] Schema name=test
[junit4] 2> 759553 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema
Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 759668 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.CoreContainer
Creating SolrCore 'collection1_shard2_replica_n61' using configuration from
collection collection1, trusted=true
[junit4] 2> 759721 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.m.r.SolrJmxReporter JMX monitoring for
'solr.core.collection1.shard2.replica_n61' (registry
'solr.core.collection1.shard2.replica_n61') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@27ab6af0
[junit4] 2> 759721 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory
solr.hdfs.home=hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home
[junit4] 2> 759727 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 759727 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore
solr.RecoveryStrategy.Builder
[junit4] 2> 759727 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore
[[collection1_shard2_replica_n61] ] Opening new SolrCore at
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_9910A7879E9A95D8-001/shard-1-001/cores/collection1_shard2_replica_n61],
dataDir=[hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/collection1/core_node62/data/]
[junit4] 2> 759728 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory creating directory factory for path
hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/collection1/core_node62/data/snapshot_metadata
[junit4] 2> 759769 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct
memory allocation set to [true]
[junit4] 2> 759769 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of
[8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 759769 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 759883 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 759952 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory creating directory factory for path
hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/collection1/core_node62/data
[junit4] 2> 760035 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory creating directory factory for path
hdfs://lucene2-us-west.apache.org:45565/solr_hdfs_home/collection1/core_node62/data/index
[junit4] 2> 760083 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct
memory allocation set to [true]
[junit4] 2> 760083 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of
[8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 760083 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 760152 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 760152 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy:
minMergeSize=1000, mergeFactor=16, maxMergeSize=9223372036854775807,
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false,
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.8471231011159144]
[junit4] 2> 760449 INFO (Block report processor) [ ] BlockStateChange
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:35176 is added to
blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null,
primaryNodeIndex=-1,
replicas=[ReplicaUC[[DISK]DS-49925226-0105-45d2-9c56-1972abb141a1:NORMAL:127.0.0.1:38952|RBW],
ReplicaUC[[DISK]DS-524ad0d4-63d5-4c10-85c9-4ca2cb6a8a90:NORMAL:127.0.0.1:35176|FINALIZED]]}
size 0
[junit4] 2> 760517 INFO (Block report processor) [ ] BlockStateChange
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38952 is added to
blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null,
primaryNodeIndex=-1,
replicas=[ReplicaUC[[DISK]DS-524ad0d4-63d5-4c10-85c9-4ca2cb6a8a90:NORMAL:127.0.0.1:35176|FINALIZED],
ReplicaUC[[DISK]DS-e93660bb-91fc-4726-8271-131628bad3aa:NORMAL:127.0.0.1:38952|FINALIZED]]}
size 0
[junit4] 2> 760650 WARN (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type =
requestHandler,name = /dump,class = DumpRequestHandler,attributes =
{initParams=a, name=/dump, class=DumpRequestHandler},args =
{defaults={a=A,b=B}}}
[junit4] 2> 760948 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateHandler
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 760948 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100
maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 760948 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.HdfsUpdateLog
Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 761048 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker
Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 761048 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker
Soft AutoCommit: disabled
[junit4] 2> 761069 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648,
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false,
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.0]
[junit4] 2> 761217 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.s.SolrIndexSearcher Opening
[Searcher@40cca79e[collection1_shard2_replica_n61] main]
[junit4] 2> 761258 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase:
/configs/conf1
[junit4] 2> 761259 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using
ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 761259 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 761260 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog
Could not find max version in index or recent updates, using new clock
1589341314794651648
[junit4] 2> 761263 INFO
(searcherExecutor-351-thread-1-processing-n:127.0.0.1:44639_
x:collection1_shard2_replica_n61 s:shard2 c:collection1) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore
[collection1_shard2_replica_n61] Registered new searcher
Searcher@40cca79e[collection1_shard2_replica_n61]
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 761276 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 761276 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 761276 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy
Sync replicas to http://127.0.0.1:44639/collection1_shard2_replica_n61/
[junit4] 2> 761276 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy
Sync Success - now sync replicas to me
[junit4] 2> 761276 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy
http://127.0.0.1:44639/collection1_shard2_replica_n61/ has no replicas
[junit4] 2> 761276 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in
election, clear LIR
[junit4] 2> 761290 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61]
o.a.s.c.ShardLeaderElectionContext I am the new leader:
http://127.0.0.1:44639/collection1_shard2_replica_n61/ shard2
[junit4] 2> 761303 INFO
(zkCallback-219-thread-1-processing-n:127.0.0.1:44639_) [n:127.0.0.1:44639_
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [2])
[junit4] 2> 761341 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ZkController
I am the leader, no recovery necessary
[junit4] 2> 761343 INFO (qtp1268555709-1381) [n:127.0.0.1:44639_
c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.HttpSolrCall
[admin] webapp=null path=/admin/cores
params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT}
status=0 QTime=3781
[junit4] 2> 761363 INFO (qtp56662695-1320) [n:127.0.0.1:42720_ ]
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections
params={node=127.0.0.1:44639_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2}
status=0 QTime=3896
[junit4] 2> 761444 INFO
(zkCallback-219-thread-1-processing-n:127.0.0.1:44639_) [n:127.0.0.1:44639_
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [2])
[junit4] 2> 761576 INFO
(OverseerCollectionConfigSetProcessor-99333831128907780-127.0.0.1:42720_-n_0000000000)
[n:127.0.0.1:42720_ ] o.a.s.c.OverseerTaskQueue Response ZK path:
/overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may
have disconnected from ZooKeeper
[junit4] 2> 762875 INFO
(TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[9910A7879E9A95D8]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkin
[...truncated too long message...]
"shared_storage":"true",
[junit4] > "state":"active",
[junit4] > "leader":"true"}}},
[junit4] > "shard2":{
[junit4] > "range":"b3330000-e665ffff",
[junit4] > "state":"active",
[junit4] > "replicas":{"core_node6":{
[junit4] >
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node6/data/",
[junit4] > "base_url":"http://127.0.0.1:39324";,
[junit4] > "node_name":"127.0.0.1:39324_",
[junit4] > "type":"NRT",
[junit4] >
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node6/data/tlog",
[junit4] > "core":"solrj_collection4_shard2_replica_n2",
[junit4] > "shared_storage":"true",
[junit4] > "state":"active",
[junit4] > "leader":"true"}}},
[junit4] > "shard3":{
[junit4] > "range":"e6660000-1998ffff",
[junit4] > "state":"active",
[junit4] > "replicas":{"core_node8":{
[junit4] > "core":"solrj_collection4_shard3_replica_n3",
[junit4] > "base_url":"http://127.0.0.1:39324";,
[junit4] > "node_name":"127.0.0.1:39324_",
[junit4] > "state":"down",
[junit4] > "type":"NRT"}}},
[junit4] > "shard4":{
[junit4] > "range":"19990000-4ccbffff",
[junit4] > "state":"active",
[junit4] > "replicas":{"core_node9":{
[junit4] >
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node9/data/",
[junit4] > "base_url":"http://127.0.0.1:39324";,
[junit4] > "node_name":"127.0.0.1:39324_",
[junit4] > "type":"NRT",
[junit4] >
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
[junit4] > "core":"solrj_collection4_shard4_replica_n5",
[junit4] > "shared_storage":"true",
[junit4] > "state":"active",
[junit4] > "leader":"true"}}},
[junit4] > "shard5":{
[junit4] > "range":"4ccc0000-7fffffff",
[junit4] > "state":"active",
[junit4] > "replicas":{"core_node10":{
[junit4] >
"dataDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node10/data/",
[junit4] > "base_url":"http://127.0.0.1:35783";,
[junit4] > "node_name":"127.0.0.1:35783_",
[junit4] > "type":"NRT",
[junit4] >
"ulogDir":"hdfs://lucene2-us-west.apache.org:40497/solr_hdfs_home/solrj_collection4/core_node10/data/tlog",
[junit4] > "core":"solrj_collection4_shard5_replica_n7",
[junit4] > "shared_storage":"true",
[junit4] > "state":"active",
[junit4] > "leader":"true"}}}},
[junit4] > "router":{
[junit4] > "field":"text",
[junit4] > "name":"compositeId"},
[junit4] > "maxShardsPerNode":"5",
[junit4] > "autoAddReplicas":"true",
[junit4] > "nrtReplicas":"1",
[junit4] > "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:35783_,
127.0.0.1:37030_, 127.0.0.1:39324_, 127.0.0.1:44676_, 127.0.0.1:45543_]
[junit4] > at
__randomizedtesting.SeedInfo.seed([9910A7879E9A95D8:1144985D3066F820]:0)
[junit4] > at
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
[junit4] > at
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:258)
[junit4] > at
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
[junit4] > at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2573049 WARN
(SUITE-SharedFSAutoReplicaFailoverTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2573057 INFO
(SUITE-SharedFSAutoReplicaFailoverTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2573158 WARN (DataNode:
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data3/,
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data4/]]
heartbeating to lucene2-us-west.apache.org/127.0.0.1:40497) [ ]
o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager
interrupted
[junit4] 2> 2573159 WARN (DataNode:
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data3/,
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data4/]]
heartbeating to lucene2-us-west.apache.org/127.0.0.1:40497) [ ]
o.a.h.h.s.d.DataNode Ending block pool service for: Block pool
BP-690830785-127.0.0.1-1515714419858 (Datanode Uuid
7756d6c1-7286-4a23-8353-898ab2fca652) service to
lucene2-us-west.apache.org/127.0.0.1:40497
[junit4] 2> 2573160 WARN
(SUITE-SharedFSAutoReplicaFailoverTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2573166 INFO
(SUITE-SharedFSAutoReplicaFailoverTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2573267 WARN (DataNode:
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data1/,
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data2/]]
heartbeating to lucene2-us-west.apache.org/127.0.0.1:40497) [ ]
o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager
interrupted
[junit4] 2> 2573268 WARN (DataNode:
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data1/,
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001/tempDir-001/hdfsBaseDir/data/data2/]]
heartbeating to lucene2-us-west.apache.org/127.0.0.1:40497) [ ]
o.a.h.h.s.d.DataNode Ending block pool service for: Block pool
BP-690830785-127.0.0.1-1515714419858 (Datanode Uuid
54b45156-79ba-49a4-b11d-07575d5c78cc) service to
lucene2-us-west.apache.org/127.0.0.1:40497
[junit4] 2> 2573289 INFO
(SUITE-SharedFSAutoReplicaFailoverTest-seed#[9910A7879E9A95D8]-worker) [ ]
o.m.log Stopped
HttpServer2$selectchannelconnectorwithsafestar...@lucene2-us-west.apache.org:0
[junit4] 2> NOTE: leaving temporary files on disk at:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.2/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9910A7879E9A95D8-001
[junit4] 2> Jan 12, 2018 12:10:03 AM
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 35 leaked
thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70):
{id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
text=Lucene50(blocksize=128)},
docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1819,
maxMBSortInHeap=5.830647428904523, sim=RandomSimilarity(queryNorm=true): {},
locale=sr, timezone=Europe/Ljubljana
[junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation
1.8.0_152 (64-bit)/cpus=4,threads=5,free=173611792,total=526909440
[junit4] 2> NOTE: All tests run in this JVM: [SolrCloudExampleTest,
TestFreeTextSuggestions, TestLFUCache, UninvertDocValuesMergePolicyTest,
TestDistributedGrouping, TestCloudDeleteByQuery, TestCharFilters,
TestConfigSets, DistributedFacetPivotSmallTest, BlockDirectoryTest,
TestFuzzyAnalyzedSuggestions, AddBlockUpdateTest, TestRequestForwarding,
TestSystemIdResolver, TestCloudPivotFacet, OverseerStatusTest, GraphQueryTest,
TestDistribDocBasedVersion, EnumFieldTest,
IgnoreCommitOptimizeUpdateProcessorFactoryTest, SpellPossibilityIteratorTest,
BadComponentTest, UnloadDistributedZkTest, TestLegacyTerms, TestOrdValues,
IndexBasedSpellCheckerTest, TriLevelCompositeIdRoutingTest,
TestLMDirichletSimilarityFactory, HLLUtilTest, RequiredFieldsTest,
TestMiniSolrCloudClusterSSL, TestLeaderInitiatedRecoveryThread,
TestDynamicLoading, SortSpecParsingTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [650/751 (2!)] on J2 in 1394.71s, 1 test, 1 failure <<<
FAILURES!
[...truncated 47623 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
