Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/401/ Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
6 tests failed. FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI Error Message: Error from server at http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6: ClusterState says we are the leader (http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6), but locally we don't think so. Request came from null Stack Trace: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6: ClusterState says we are the leader (http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6), but locally we don't think so. Request came from null at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:D5AE5A5CFCFAE205]:0) at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:550) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1013) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6: ClusterState says we are the leader (http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6), but locally we don't think so. Request came from null at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:527) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more FAILED: org.apache.solr.cloud.TestPullReplica.testRealTimeGet Error Message: Unexpected replica count null Live Nodes: [127.0.0.1:59416_solr, 127.0.0.1:59417_solr] Last available state: null Stack Trace: java.lang.AssertionError: Unexpected replica count null Live Nodes: [127.0.0.1:59416_solr, 127.0.0.1:59417_solr] Last available state: null at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:C5B6DBEB58736359]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269) at org.apache.solr.cloud.TestPullReplica.testRealTimeGet(TestPullReplica.java:329) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete {seed=[9DDB2EE8FAC9CD90:86CBDFA40C3935A0]} Error Message: Could not find collection : pull_replica_test_create_delete Stack Trace: org.apache.solr.common.SolrException: Could not find collection : pull_replica_test_create_delete at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:86CBDFA40C3935A0]:0) at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:118) at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247) at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:162) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete {seed=[9DDB2EE8FAC9CD90:329D635838FBFE8C]} Error Message: Too many open files Stack Trace: java.net.SocketException: Too many open files at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:329D635838FBFE8C]:0) at java.net.Socket.createImpl(Socket.java:460) at java.net.Socket.getImpl(Socket.java:520) at java.net.Socket.setSoTimeout(Socket.java:1141) at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:120) at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359) at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:156) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.search.join.TestCloudNestedDocsSort Error Message: Could not find collection:collection1 Stack Trace: java.lang.AssertionError: Could not find collection:collection1 at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNotNull(Assert.java:526) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155) at org.apache.solr.search.join.TestCloudNestedDocsSort.setupCluster(TestCloudNestedDocsSort.java:75) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.servlet.HttpSolrCallGetCoreTest Error Message: Could not find collection:collection1 Stack Trace: java.lang.AssertionError: Could not find collection:collection1 at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNotNull(Assert.java:526) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155) at org.apache.solr.servlet.HttpSolrCallGetCoreTest.setupCluster(HttpSolrCallGetCoreTest.java:52) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 13207 lines...] [junit4] Suite: org.apache.solr.cloud.TestPullReplica [junit4] 2> 3112347 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/init-core-data-001 [junit4] 2> 3112348 WARN (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2 [junit4] 2> 3112348 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false [junit4] 2> 3112355 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth [junit4] 2> 3112356 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001 [junit4] 2> 3112356 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 3112356 INFO (Thread-3126) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 3112356 INFO (Thread-3126) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 3112359 ERROR (Thread-3126) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 3112465 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:59414 [junit4] 2> 3112481 INFO (zkConnectionManagerCallback-3595-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112490 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 3112490 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4acb9b0f{/solr,null,AVAILABLE} [junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@642dbe{/solr,null,AVAILABLE} [junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6c230b37{HTTP/1.1,[http/1.1]}{127.0.0.1:59417} [junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.Server Started @3116315ms [junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59417} [junit4] 2> 3112495 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3f540ccf{HTTP/1.1,[http/1.1]}{127.0.0.1:59416} [junit4] 2> 3112495 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.Server Started @3116317ms [junit4] 2> 3112495 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59416} [junit4] 2> 3112495 ERROR (jetty-launcher-3592-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 3112495 ERROR (jetty-launcher-3592-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0 [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0 [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T15:47:05.222Z [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T15:47:05.222Z [junit4] 2> 3112498 INFO (zkConnectionManagerCallback-3597-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112498 INFO (zkConnectionManagerCallback-3599-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112499 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 3112499 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 3112528 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59414/solr [junit4] 2> 3112528 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59414/solr [junit4] 2> 3112530 INFO (zkConnectionManagerCallback-3605-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112530 INFO (zkConnectionManagerCallback-3607-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112533 INFO (zkConnectionManagerCallback-3610-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112533 INFO (zkConnectionManagerCallback-3611-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112711 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 3112712 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 3112712 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59416_solr [junit4] 2> 3112713 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59417_solr [junit4] 2> 3112714 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.Overseer Overseer (id=99354622367105029-127.0.0.1:59416_solr-n_0000000000) starting [junit4] 2> 3112717 INFO (zkCallback-3608-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 3112719 INFO (zkCallback-3609-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 3112728 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59416_solr [junit4] 2> 3112730 INFO (zkCallback-3609-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 3112730 INFO (zkCallback-3608-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 3112818 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3112824 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3112824 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3112825 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/. [junit4] 2> 3112861 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3112867 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3112867 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3112868 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/. [junit4] 2> 3112891 INFO (zkConnectionManagerCallback-3617-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112894 INFO (zkConnectionManagerCallback-3621-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 3112896 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 3112897 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59414/solr ready [junit4] 2> 3112903 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.TestPullReplica Using legacyCloud?: false [junit4] 2> 3112907 INFO (qtp1042818031-17484) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 3112908 INFO (qtp1042818031-17484) [n:127.0.0.1:59416_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1 [junit4] 2> 3113016 INFO (TEST-TestPullReplica.testRealTimeGet-seed#[9DDB2EE8FAC9CD90]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRealTimeGet [junit4] 2> 3113018 INFO (qtp1042818031-17486) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=2&replicationFactor=2&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_real_time_get&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 3113021 INFO (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_real_time_get [junit4] 2> 3113022 WARN (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_real_time_get is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance. [junit4] 2> 3113136 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_real_time_get", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:59416/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 3113140 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_real_time_get", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_n2", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:59417/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 3113144 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_real_time_get", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_p4", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:59416/solr";, [junit4] 2> "type":"PULL", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 3113148 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_real_time_get", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_p6", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:59417/solr";, [junit4] 2> "type":"PULL", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 3113357 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p4&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 3113357 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n2&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT [junit4] 2> 3113358 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 3113358 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 3113358 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p6&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 3113358 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT [junit4] 2> 3113472 INFO (zkCallback-3608-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2]) [junit4] 2> 3113472 INFO (zkCallback-3609-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2]) [junit4] 2> 3113472 INFO (zkCallback-3608-thread-2-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2]) [junit4] 2> 3113473 INFO (zkCallback-3609-thread-2-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2]) [junit4] 2> 3114390 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0 [junit4] 2> 3114391 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0 [junit4] 2> 3114403 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0 [junit4] 2> 3114403 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0 [junit4] 2> 3114408 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_p6] Schema name=minimal [junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6' using configuration from collection pull_replica_test_real_time_get, trusted=true [junit4] 2> 3114408 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_p4] Schema name=minimal [junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.core.pull_replica_test_real_time_get.shard1.replica_p6' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_p6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/pull_replica_test_real_time_get_shard1_replica_p6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/./pull_replica_test_real_time_get_shard1_replica_p6/data/] [junit4] 2> 3114413 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 3114413 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_p4' using configuration from collection pull_replica_test_real_time_get, trusted=true [junit4] 2> 3114414 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.core.pull_replica_test_real_time_get.shard1.replica_p4' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3114414 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 3114414 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_p4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/pull_replica_test_real_time_get_shard1_replica_p4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/./pull_replica_test_real_time_get_shard1_replica_p4/data/] [junit4] 2> 3114416 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_n2] Schema name=minimal [junit4] 2> 3114417 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_n1] Schema name=minimal [junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2' using configuration from collection pull_replica_test_real_time_get, trusted=true [junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.core.pull_replica_test_real_time_get.shard1.replica_n2' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/pull_replica_test_real_time_get_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/./pull_replica_test_real_time_get_shard1_replica_n2/data/] [junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1' using configuration from collection pull_replica_test_real_time_get, trusted=true [junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.core.pull_replica_test_real_time_get.shard1.replica_n1' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d [junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/pull_replica_test_real_time_get_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/./pull_replica_test_real_time_get_shard1_replica_n1/data/] [junit4] 2> 3114506 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@190e5b52 [junit4] 2> 3114506 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_n2, tag=420371282 [junit4] 2> 3114509 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5a7480b5: rootName = solr_59417, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@7b47a79d [junit4] 2> 3114509 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3dfdb0dc [junit4] 2> 3114509 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_n1, tag=1040036060 [junit4] 2> 3114509 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48f77088: rootName = solr_59416, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@41d2887a [junit4] 2> 3114510 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@35e4ce9f [junit4] 2> 3114512 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p6] CLOSING SolrCore org.apache.solr.core.SolrCore@7fb6c75f [junit4] 2> 3114512 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=420371282 [junit4] 2> 3114513 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_p6, tag=2142685023 [junit4] 2> 3114513 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7296ee99: rootName = solr_59417, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_p6, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_p6 / com.codahale.metrics.MetricRegistry@7d7c988e [junit4] 2> 3114515 ERROR (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n2] Caused by: invalid API spec: apispec/core.SchemaEdit.json [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) [junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) [junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) [junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534) [junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) [junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) [junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) [junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) [junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) [junit4] 2> at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_n2] [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1055) [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950) [junit4] 2> ... 35 more [junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1008) [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863) [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039) [junit4] 2> ... 36 more [junit4] 2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json [junit4] 2> at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318) [junit4] 2> at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:427) [junit4] 2> at org.apache.solr.api.Api.getSpec(Api.java:65) [junit4] 2> at org.apache.solr.api.ApiBag.register(ApiBag.java:73) [junit4] 2> at org.apache.solr.core.PluginBag.put(PluginBag.java:217) [junit4] 2> at org.apache.solr.core.PluginBag.init(PluginBag.java:274) [junit4] 2> at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130) [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:966) [junit4] 2> ... 38 more [junit4] 2> [junit4] 2> 3114515 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n2&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=400 QTime=1157 [junit4] 2> 3114516 ERROR (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:59417/solr [junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59417/solr: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n2] Caused by: invalid API spec: apispec/core.SchemaEdit.json [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) [junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) [junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 3114521 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=1040036060 [junit4] 2> 3114523 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_p4, tag=904187551 [junit4] 2> 3114524 ERROR (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n1] Caused by: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) [junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) [junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) [junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534) [junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) [junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) [junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) [junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) [junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_n1] [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1055) [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950) [junit4] 2> ... 34 more [junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1008) [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863) [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039) [junit4] 2> ... 35 more [junit4] 2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318) [junit4] 2> at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:427) [junit4] 2> at org.apache.solr.api.Api.getSpec(Api.java:65) [junit4] 2> at org.apache.solr.api.ApiBag.register(ApiBag.java:73) [junit4] 2> at org.apache.solr.core.PluginBag.put(PluginBag.java:217) [junit4] 2> at org.apache.solr.core.PluginBag.init(PluginBag.java:274) [junit4] 2> at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130) [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:966) [junit4] 2> ... 37 more [junit4] 2> [junit4] 2> 3114524 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=400 QTime=1166 [junit4] 2> 3114525 ERROR (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:59416/solr [junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59416/solr: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n1] Caused by: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) [junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) [junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 3114528 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2c3163f3: rootName = solr_59416, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_p4, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_p4 / com.codahale.metrics.MetricRegistry@7f25f41e [junit4] 2> 3114528 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=2142685023 [junit4] 2> 3114529 ERROR (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6': Unable to create core [pull_replica_test_real_time_get_shard1_replica_p6] Caused by: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) [junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) [junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) [junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534) [junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) [junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) [junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) [junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) [junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_p6] [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1055) [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950) [junit4] 2> ... 34 more [junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1008) [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863) [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039) [junit4] 2> ... 35 more [junit4] 2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318) [junit4] 2> at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:427) [junit4] 2> at org.apache.solr.api.Api.getSpec(Api.java:65) [junit4] 2> at org.apache.solr.api.ApiBag.register(ApiBag.java:73) [junit4] 2> at org.apache.solr.core.PluginBag.put(PluginBag.java:217) [junit4] 2> at org.apache.solr.core.PluginBag.init(PluginBag.java:274) [junit4] 2> at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130) [junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:966) [junit4] 2> ... 37 more [junit4] 2> [junit4] 2> 3114530 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p6&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=400 QTime=1171 [junit4] 2> 3114530 ERROR (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:59417/solr [junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59417/solr: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6': Unable to create core [pull_replica_test_real_time_get_shard1_replica_p6] Caused by: invalid API spec: apispec/emptySpec.json [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) [junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) [junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) [junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 3114533 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=904187551 [junit4] 2> 3114537 ERROR (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_p4': Unable to create core [pull_replica_test_real_time_get_shard1_replica_p4] Caused by: invalid API spec: apispec/core.SchemaEdit.json [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) [junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) [junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) [junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534) [junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) [junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) [junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) [junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) [junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_p4] [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:105 [...truncated too long message...] [junit4] 2> 3644771 INFO (jetty-closer-4002-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null [junit4] 2> 3644772 INFO (jetty-closer-4002-thread-4) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60464_solr [junit4] 2> 3644773 INFO (zkCallback-4029-thread-3-processing-n:127.0.0.1:60464_solr) [n:127.0.0.1:60464_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2) [junit4] 2> 3644773 INFO (zkCallback-4070-thread-2-processing-n:127.0.0.1:60463_solr) [n:127.0.0.1:60463_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2) [junit4] 2> 3644773 INFO (zkCallback-4031-thread-7-processing-n:127.0.0.1:60465_solr) [n:127.0.0.1:60465_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2) [junit4] 2> 3644773 INFO (zkCallback-4035-thread-4-processing-n:127.0.0.1:60466_solr) [n:127.0.0.1:60466_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2) [junit4] 2> 3644776 INFO (jetty-closer-4002-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@73e6ceb4{/solr,null,UNAVAILABLE} [junit4] 2> 3644778 INFO (zkCallback-4070-thread-6-processing-n:127.0.0.1:60463_solr) [n:127.0.0.1:60463_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1) [junit4] 2> 3644779 INFO (zkCallback-4029-thread-1-processing-n:127.0.0.1:60464_solr) [n:127.0.0.1:60464_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1) [junit4] 2> 3644780 INFO (jetty-closer-4002-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@30c66fd8{/solr,null,UNAVAILABLE} [junit4] 2> 3644783 INFO (jetty-closer-4002-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60463_solr [junit4] 2> 3644783 INFO (jetty-closer-4002-thread-4) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null [junit4] 2> 3644784 INFO (zkCallback-4070-thread-6-processing-n:127.0.0.1:60463_solr) [n:127.0.0.1:60463_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 3644785 INFO (zkCallback-4029-thread-1-processing-n:127.0.0.1:60464_solr) [n:127.0.0.1:60464_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 3644786 INFO (jetty-closer-4002-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null [junit4] 2> 3644786 INFO (jetty-closer-4002-thread-4) [ ] o.a.s.c.Overseer Overseer (id=99354647925817353-127.0.0.1:60464_solr-n_0000000000) closing [junit4] 2> 3644787 INFO (OverseerStateUpdate-99354647925817353-127.0.0.1:60464_solr-n_0000000000) [n:127.0.0.1:60464_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60464_solr [junit4] 2> 3644789 INFO (jetty-closer-4002-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3d68c24c{/solr,null,UNAVAILABLE} [junit4] 2> 3644789 WARN (OverseerAutoScalingTriggerThread-99354647925817353-127.0.0.1:60464_solr-n_0000000000) [n:127.0.0.1:60464_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 3644798 INFO (jetty-closer-4002-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6c85c338{/solr,null,UNAVAILABLE} [junit4] 2> 3644799 ERROR (SUITE-CollectionsAPIDistributedZkTest-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 3644800 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60461 60461 [junit4] 2> 3644803 INFO (Thread-3419) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60461 60461 [junit4] 2> 3645000 WARN (Thread-3419) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 9 /solr/aliases.json [junit4] 2> 9 /solr/clusterprops.json [junit4] 2> 7 /solr/configs/conf [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 15 /solr/collections/acollectionafterbaddelete/state.json [junit4] 2> 13 /solr/collections/onlyinzk/state.json [junit4] 2> 10 /solr/collections/awhollynewcollection_0/state.json [junit4] 2> 9 /solr/clusterstate.json [junit4] 2> 5 /solr/collections/halfcollectionblocker/state.json [junit4] 2> 5 /solr/collections/halfcollection/state.json [junit4] 2> 5 /solr/collections/created_and_deleted/state.json [junit4] 2> 5 /solr/collections/withconfigset2/state.json [junit4] 2> 5 /solr/collections/halfcollectionblocker2/state.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 324 /solr/collections [junit4] 2> 9 /solr/live_nodes [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_9DDB2EE8FAC9CD90-001 [junit4] 2> Jan 15, 2018 3:55:57 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=sv-SE, timezone=America/Atka [junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=137806224,total=467017728 [junit4] 2> NOTE: All tests run in this JVM: [TestFastWriter, TestExactSharedStatsCache, TestConfig, V2ApiIntegrationTest, CoreAdminRequestStatusTest, TestPostingsSolrHighlighter, DeleteLastCustomShardedReplicaTest, CdcrReplicationHandlerTest, TestLegacyTerms, RecoveryAfterSoftCommitTest, SystemInfoHandlerTest, ComputePlanActionTest, ZkSolrClientTest, TestComplexPhraseLeadingWildcard, ZkNodePropsTest, TestExecutePlanAction, TestFieldCacheWithThreads, TestValueSourceCache, TestSchemaManager, FileUtilsTest, TestExtendedDismaxParser, HdfsNNFailoverTest, TestQueryWrapperFilter, TestReload, TestBadConfig, DebugComponentTest, BasicDistributedZk2Test, TestInfoStreamLogging, QueryEqualityTest, TestSuggestSpellingConverter, TestInPlaceUpdatesDistrib, TestLargeCluster, DeleteReplicaTest, TestNodeLostTrigger, MetricUtilsTest, TestRuleBasedAuthorizationPlugin, TestBlobHandler, SystemLogListenerTest, TestCrossCoreJoin, TestStressLucene, RequestLoggingTest, TestJettySolrRunner, TestRequestForwarding, FieldMutatingUpdateProcessorTest, PingRequestHandlerTest, TestCollectionsAPIViaSolrCloudCluster, TermVectorComponentDistributedTest, TestSort, TestDocTermOrds, TestBM25SimilarityFactory, TestElisionMultitermQuery, DistributedQueryComponentOptimizationTest, TestUtilizeNode, SearchHandlerTest, SpatialHeatmapFacetsTest, TestCloudManagedSchema, TestDefaultStatsCache, DocValuesMissingTest, BasicFunctionalityTest, HdfsAutoAddReplicasIntegrationTest, TestSolrConfigHandlerConcurrent, TestStressReorder, SolrCoreTest, TestConfigSetsAPIExclusivity, TestDFISimilarityFactory, TestStressCloudBlindAtomicUpdates, UninvertDocValuesMergePolicyTest, OverseerCollectionConfigSetProcessorTest, TestPivotHelperCode, UpdateParamsTest, ZkFailoverTest, DistributedFacetPivotLargeTest, TestMinMaxOnMultiValuedField, TestSimpleTrackingShardHandler, ChaosMonkeyNothingIsSafeWithPullReplicasTest, MultiThreadedOCPTest, TestBulkSchemaAPI, TestFieldTypeCollectionResource, TestGroupingSearch, TestClusterStateMutator, TestAnalyzeInfixSuggestions, TestHighFrequencyDictionaryFactory, TestQueryUtils, SegmentsInfoRequestHandlerTest, SSLMigrationTest, TestTrie, TestNRTOpen, HdfsTlogReplayBufferedWhileIndexingTest, TestRTGBase, LeaderElectionIntegrationTest, TestMacroExpander, TriggerIntegrationTest, ConjunctionSolrSpellCheckerTest, SolrIndexConfigTest, TestRebalanceLeaders, TestZkChroot, TermVectorComponentTest, TestJsonRequest, SolrPluginUtilsTest, CollectionsAPIAsyncDistributedZkTest, PluginInfoTest, TestConfigOverlay, AliasIntegrationTest, TestExportWriter, AnalysisAfterCoreReloadTest, LeaderInitiatedRecoveryOnCommitTest, TestUseDocValuesAsStored, TestLeaderInitiatedRecoveryThread, LoggingHandlerTest, SuggestComponentTest, CurrencyRangeFacetCloudTest, InfoHandlerTest, TestNumericRangeQuery32, ZkStateWriterTest, TestLuceneIndexBackCompat, TestSolrCloudWithKerberosAlt, TestPerFieldSimilarity, TestCloudPivotFacet, TestSolrQueryParser, TestApiFramework, SubstringBytesRefFilterTest, TestTrieFacet, OpenCloseCoreStressTest, TestMergePolicyConfig, SolrSlf4jReporterTest, TestSSLRandomization, SpatialRPTFieldTypeTest, TestShortCircuitedRequests, TestSQLHandlerNonCloud, SimplePostToolTest, BaseCdcrDistributedZkTest, DistributedMLTComponentTest, TestSubQueryTransformerDistrib, CdcrBootstrapTest, BooleanFieldTest, TestDocSet, AssignTest, FullSolrCloudDistribCmdsTest, DocValuesNotIndexedTest, BlobRepositoryCloudTest, FacetPivotSmallTest, TestAddFieldRealTimeGet, DistributedQueueTest, TestSolrDeletionPolicy1, AddSchemaFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, TestPullReplica, TemplateUpdateProcessorTest, HttpSolrCallGetCoreTest, UUIDUpdateProcessorFallbackTest, TestCloudNestedDocsSort, StatelessScriptUpdateProcessorFactoryTest, TestComponentsName, TestCharFilters, TestDynamicLoading, TestFaceting, SynonymTokenizerTest, CurrencyFieldTypeTest, TestFastOutputStream, CollectionsAPIDistributedZkTest] [junit4] Completed [499/767 (4!)] on J1 in 143.19s, 19 tests, 1 error <<< FAILURES! [...truncated 49042 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
