Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/367/
1 tests failed. FAILED: org.apache.solr.cloud.hdfs.StressHdfsTest.test Error Message: Timeout occured while waiting response from server at: http://127.0.0.1:51723 Stack Trace: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:51723 at __randomizedtesting.SeedInfo.seed([3531B02B79E0B1B4:BD658FF1D71CDC4C]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:623) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:477) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:407) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1383) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1134) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1073) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:220) at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160) at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84) at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261) at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283) at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251) at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124) at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487) at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:515) ... 51 more Build Log: [...truncated 11989 lines...] [junit4] Suite: org.apache.solr.cloud.hdfs.StressHdfsTest [junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/init-core-data-001 [junit4] 2> 760701 WARN (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5 [junit4] 2> 760701 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields [junit4] 2> 760704 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) [junit4] 2> 760704 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: / [junit4] 1> Formatting using clusterid: testClusterID [junit4] 2> 760782 WARN (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties [junit4] 2> 760805 WARN (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j [junit4] 2> 760808 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log jetty-6.1.26 [junit4] 2> 760838 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_58150_hdfs____at8l2r/webapp [junit4] 2> 761263 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:58150 [junit4] 2> 761453 WARN (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j [junit4] 2> 761454 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log jetty-6.1.26 [junit4] 2> 761482 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_52413_datanode____oewg00/webapp [junit4] 2> 762189 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:52413 [junit4] 2> 762245 WARN (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j [junit4] 2> 762246 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log jetty-6.1.26 [junit4] 2> 762262 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_60404_datanode____.3u97tn/webapp [junit4] 2> 762360 INFO (IPC Server handler 3 on 40142) [ ] BlockStateChange BLOCK* processReport: from storage DS-308c8e3a-0af2-481b-9eeb-420d330ffeab node DatanodeRegistration(127.0.0.1:54611, datanodeUuid=86eaabeb-e1b1-4ba9-a930-806276ff60c4, infoPort=41939, infoSecurePort=0, ipcPort=58284, storageInfo=lv=-56;cid=testClusterID;nsid=1173970763;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs [junit4] 2> 762361 INFO (IPC Server handler 3 on 40142) [ ] BlockStateChange BLOCK* processReport: from storage DS-76a51a6e-4058-4bba-9041-81d141ca2c2d node DatanodeRegistration(127.0.0.1:54611, datanodeUuid=86eaabeb-e1b1-4ba9-a930-806276ff60c4, infoPort=41939, infoSecurePort=0, ipcPort=58284, storageInfo=lv=-56;cid=testClusterID;nsid=1173970763;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs [junit4] 2> 762755 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:60404 [junit4] 2> 763012 INFO (IPC Server handler 5 on 40142) [ ] BlockStateChange BLOCK* processReport: from storage DS-7ef2b07b-693e-4c73-860f-e4e7f3a1aa3b node DatanodeRegistration(127.0.0.1:38092, datanodeUuid=daeebff5-13f3-4131-ab6d-2c62946773bc, infoPort=36673, infoSecurePort=0, ipcPort=44470, storageInfo=lv=-56;cid=testClusterID;nsid=1173970763;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs [junit4] 2> 763012 INFO (IPC Server handler 5 on 40142) [ ] BlockStateChange BLOCK* processReport: from storage DS-2e7827eb-ab45-4dca-b444-8b82af11d5d8 node DatanodeRegistration(127.0.0.1:38092, datanodeUuid=daeebff5-13f3-4131-ab6d-2c62946773bc, infoPort=36673, infoSecurePort=0, ipcPort=44470, storageInfo=lv=-56;cid=testClusterID;nsid=1173970763;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs [junit4] 2> 763154 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 763155 INFO (Thread-2389) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 763155 INFO (Thread-2389) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 763161 ERROR (Thread-2389) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 763254 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ZkTestServer start zk server on port:38291 [junit4] 2> 763269 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml [junit4] 2> 763271 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml [junit4] 2> 763273 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 763274 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt [junit4] 2> 763275 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt [junit4] 2> 763277 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml [junit4] 2> 763278 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml [junit4] 2> 763280 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json [junit4] 2> 763283 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt [junit4] 2> 763289 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt [junit4] 2> 763291 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt [junit4] 2> 763403 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/control-001/cores/collection1 [junit4] 2> 763405 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028 [junit4] 2> 763406 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24eb264{/,null,AVAILABLE} [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@72fd3ae8{HTTP/1.1,[http/1.1]}{127.0.0.1:60555} [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server Started @766445ms [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40142/hdfs__localhost_40142__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001_tempDir-002_control_data, hostContext=/, hostPort=60555, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/control-001/cores} [junit4] 2> 763407 ERROR (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0 [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 763407 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-06-10T23:21:46.045Z [junit4] 2> 763420 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 763420 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/control-001/solr.xml [junit4] 2> 763427 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 763428 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38291/solr [junit4] 2> 763469 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:60555_ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 763470 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:60555_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60555_ [junit4] 2> 763475 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:60555_ ] o.a.s.c.Overseer Overseer (id=98116364258181124-127.0.0.1:60555_-n_0000000000) starting [junit4] 2> 763488 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:60555_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60555_ [junit4] 2> 763502 INFO (zkCallback-1266-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 763569 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:60555_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/control-001/cores [junit4] 2> 763569 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:60555_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1] [junit4] 2> 763570 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores [junit4] 2> 763575 INFO (OverseerStateUpdate-98116364258181124-127.0.0.1:60555_-n_0000000000) [n:127.0.0.1:60555_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 764600 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0 [junit4] 2> 764624 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test [junit4] 2> 764782 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id [junit4] 2> 764809 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection, trusted=true [junit4] 2> 764810 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40142/solr_hdfs_home [junit4] 2> 764810 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled [junit4] 2> 764810 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 764810 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/control-001/cores/collection1], dataDir=[hdfs://localhost:40142/solr_hdfs_home/control_collection/core_node1/data/] [junit4] 2> 764810 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2254e2 [junit4] 2> 764810 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata [junit4] 2> 764819 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 764819 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 764819 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 764833 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 764834 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/control_collection/core_node1/data [junit4] 2> 764870 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/control_collection/core_node1/data/index [junit4] 2> 764880 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 764881 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 764881 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 764894 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 764894 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=74.70703125, floorSegmentMB=1.3876953125, forceMergeDeletesPctAllowed=12.7689070004221, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1777327330543834 [junit4] 2> 764970 INFO (IPC Server handler 9 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:54611 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-2e7827eb-ab45-4dca-b444-8b82af11d5d8:NORMAL:127.0.0.1:38092|RBW], ReplicaUC[[DISK]DS-76a51a6e-4058-4bba-9041-81d141ca2c2d:NORMAL:127.0.0.1:54611|RBW]]} size 0 [junit4] 2> 764972 INFO (IPC Server handler 1 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38092 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-76a51a6e-4058-4bba-9041-81d141ca2c2d:NORMAL:127.0.0.1:54611|RBW], ReplicaUC[[DISK]DS-7ef2b07b-693e-4c73-860f-e4e7f3a1aa3b:NORMAL:127.0.0.1:38092|FINALIZED]]} size 0 [junit4] 2> 765000 WARN (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}} [junit4] 2> 765142 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog [junit4] 2> 765142 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536 [junit4] 2> 765142 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2 [junit4] 2> 765155 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 765155 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 765162 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=69.03125, floorSegmentMB=1.478515625, forceMergeDeletesPctAllowed=19.904408526357635, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0 [junit4] 2> 765193 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5df54d1e[collection1] main] [junit4] 2> 765194 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1 [junit4] 2> 765195 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 765195 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 765196 INFO (searcherExecutor-3052-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5df54d1e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 765199 INFO (coreLoadExecutor-3051-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1569861830272090112 [junit4] 2> 765229 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. [junit4] 2> 765230 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync [junit4] 2> 765230 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60555/collection1/ [junit4] 2> 765231 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me [junit4] 2> 765231 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60555/collection1/ has no replicas [junit4] 2> 765231 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR [junit4] 2> 765236 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60555/collection1/ shard1 [junit4] 2> 765387 INFO (coreZkRegister-3044-thread-1-processing-n:127.0.0.1:60555_ x:collection1 c:control_collection) [n:127.0.0.1:60555_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary [junit4] 2> 765605 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 765608 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38291/solr ready [junit4] 2> 765608 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false [junit4] 2> 765754 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-1-001/cores/collection1 [junit4] 2> 765755 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-1-001 [junit4] 2> 765757 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028 [junit4] 2> 765760 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36c99beb{/,null,AVAILABLE} [junit4] 2> 765760 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@22cc55db{HTTP/1.1,[http/1.1]}{127.0.0.1:54207} [junit4] 2> 765760 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server Started @768798ms [junit4] 2> 765760 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40142/hdfs__localhost_40142__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=54207, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-1-001/cores} [junit4] 2> 765761 ERROR (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 765769 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0 [junit4] 2> 765769 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 765769 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 765769 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-06-10T23:21:48.407Z [junit4] 2> 765775 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 765775 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-1-001/solr.xml [junit4] 2> 765783 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 765784 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38291/solr [junit4] 2> 765814 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:54207_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 765828 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:54207_ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 765833 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:54207_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54207_ [junit4] 2> 765836 INFO (zkCallback-1270-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 765837 INFO (zkCallback-1266-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 765838 INFO (zkCallback-1275-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 765918 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:54207_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-1-001/cores [junit4] 2> 765918 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:54207_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1] [junit4] 2> 765919 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores [junit4] 2> 765925 INFO (OverseerStateUpdate-98116364258181124-127.0.0.1:60555_-n_0000000000) [n:127.0.0.1:60555_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 766951 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0 [junit4] 2> 766979 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test [junit4] 2> 767138 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id [junit4] 2> 767165 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true [junit4] 2> 767166 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40142/solr_hdfs_home [junit4] 2> 767166 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled [junit4] 2> 767166 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 767167 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-1-001/cores/collection1], dataDir=[hdfs://localhost:40142/solr_hdfs_home/collection1/core_node1/data/] [junit4] 2> 767168 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2254e2 [junit4] 2> 767168 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata [junit4] 2> 767183 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 767183 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 767184 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 767213 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 767216 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node1/data [junit4] 2> 767257 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node1/data/index [junit4] 2> 767269 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 767269 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 767269 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 767281 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 767282 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=74.70703125, floorSegmentMB=1.3876953125, forceMergeDeletesPctAllowed=12.7689070004221, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1777327330543834 [junit4] 2> 767344 INFO (IPC Server handler 4 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38092 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-308c8e3a-0af2-481b-9eeb-420d330ffeab:NORMAL:127.0.0.1:54611|RBW], ReplicaUC[[DISK]DS-2e7827eb-ab45-4dca-b444-8b82af11d5d8:NORMAL:127.0.0.1:38092|FINALIZED]]} size 0 [junit4] 2> 767347 INFO (IPC Server handler 5 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:54611 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-308c8e3a-0af2-481b-9eeb-420d330ffeab:NORMAL:127.0.0.1:54611|RBW], ReplicaUC[[DISK]DS-2e7827eb-ab45-4dca-b444-8b82af11d5d8:NORMAL:127.0.0.1:38092|FINALIZED]]} size 0 [junit4] 2> 767403 WARN (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}} [junit4] 2> 767544 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog [junit4] 2> 767544 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536 [junit4] 2> 767544 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2 [junit4] 2> 767560 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 767560 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 767564 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=69.03125, floorSegmentMB=1.478515625, forceMergeDeletesPctAllowed=19.904408526357635, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0 [junit4] 2> 767592 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@40ca1c55[collection1] main] [junit4] 2> 767594 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1 [junit4] 2> 767594 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 767598 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 767601 INFO (searcherExecutor-3063-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@40ca1c55[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 767615 INFO (coreLoadExecutor-3062-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1569861832805449728 [junit4] 2> 767647 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. [junit4] 2> 767647 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync [junit4] 2> 767647 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54207/collection1/ [junit4] 2> 767647 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me [junit4] 2> 767647 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:54207/collection1/ has no replicas [junit4] 2> 767647 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR [junit4] 2> 767657 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54207/collection1/ shard1 [junit4] 2> 767762 INFO (coreZkRegister-3057-thread-1-processing-n:127.0.0.1:54207_ x:collection1 c:collection1) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary [junit4] 2> 768061 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-2-001/cores/collection1 [junit4] 2> 768062 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-2-001 [junit4] 2> 768063 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028 [junit4] 2> 768064 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5bf8fbe5{/,null,AVAILABLE} [junit4] 2> 768065 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@23cf9638{HTTP/1.1,[http/1.1]}{127.0.0.1:34386} [junit4] 2> 768065 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server Started @771102ms [junit4] 2> 768065 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40142/hdfs__localhost_40142__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001_tempDir-002_jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=34386, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-2-001/cores} [junit4] 2> 768065 ERROR (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 768069 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0 [junit4] 2> 768069 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 768069 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 768069 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-06-10T23:21:50.707Z [junit4] 2> 768081 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 768081 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-2-001/solr.xml [junit4] 2> 768088 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 768092 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38291/solr [junit4] 2> 768134 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:34386_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 768136 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:34386_ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 768138 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:34386_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34386_ [junit4] 2> 768141 INFO (zkCallback-1270-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 768141 INFO (zkCallback-1275-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 768145 INFO (zkCallback-1266-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 768145 INFO (zkCallback-1281-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 768284 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:34386_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-2-001/cores [junit4] 2> 768284 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:34386_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1] [junit4] 2> 768289 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores [junit4] 2> 768291 INFO (OverseerStateUpdate-98116364258181124-127.0.0.1:60555_-n_0000000000) [n:127.0.0.1:60555_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 769346 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0 [junit4] 2> 769367 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test [junit4] 2> 769561 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id [junit4] 2> 769584 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true [junit4] 2> 769585 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40142/solr_hdfs_home [junit4] 2> 769585 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled [junit4] 2> 769585 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 769585 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-2-001/cores/collection1], dataDir=[hdfs://localhost:40142/solr_hdfs_home/collection1/core_node2/data/] [junit4] 2> 769585 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2254e2 [junit4] 2> 769585 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata [junit4] 2> 769597 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 769597 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 769597 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 769617 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 769618 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node2/data [junit4] 2> 769643 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node2/data/index [junit4] 2> 769653 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 769653 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 769653 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 769665 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 769666 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=74.70703125, floorSegmentMB=1.3876953125, forceMergeDeletesPctAllowed=12.7689070004221, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1777327330543834 [junit4] 2> 769730 INFO (IPC Server handler 1 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:54611 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-2e7827eb-ab45-4dca-b444-8b82af11d5d8:NORMAL:127.0.0.1:38092|RBW], ReplicaUC[[DISK]DS-76a51a6e-4058-4bba-9041-81d141ca2c2d:NORMAL:127.0.0.1:54611|RBW]]} size 0 [junit4] 2> 769731 INFO (IPC Server handler 0 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38092 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-76a51a6e-4058-4bba-9041-81d141ca2c2d:NORMAL:127.0.0.1:54611|RBW], ReplicaUC[[DISK]DS-7ef2b07b-693e-4c73-860f-e4e7f3a1aa3b:NORMAL:127.0.0.1:38092|FINALIZED]]} size 0 [junit4] 2> 769759 WARN (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}} [junit4] 2> 769870 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog [junit4] 2> 769870 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536 [junit4] 2> 769870 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2 [junit4] 2> 769897 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 769897 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 769922 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=69.03125, floorSegmentMB=1.478515625, forceMergeDeletesPctAllowed=19.904408526357635, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0 [junit4] 2> 769939 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@25834241[collection1] main] [junit4] 2> 769944 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1 [junit4] 2> 769946 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 769946 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 769963 INFO (searcherExecutor-3074-thread-1-processing-n:127.0.0.1:34386_ x:collection1 c:collection1) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@25834241[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 769982 INFO (coreLoadExecutor-3073-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1569861835287429120 [junit4] 2> 769992 INFO (coreZkRegister-3068-thread-1-processing-n:127.0.0.1:34386_ x:collection1 c:collection1) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1 [junit4] 2> 769992 INFO (updateExecutor-1278-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery [junit4] 2> 769993 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true [junit4] 2> 769994 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]] [junit4] 2> 769994 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1] [junit4] 2> 769994 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. HDFSUpdateLog{state=ACTIVE, tlog=null} [junit4] 2> 769994 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:54207/collection1/] and I am [http://127.0.0.1:34386/collection1/] [junit4] 2> 769999 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:54207]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:34386_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true] [junit4] 2> 770040 INFO (qtp1855766365-8317) [n:127.0.0.1:54207_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true [junit4] 2> 770041 INFO (qtp1855766365-8317) [n:127.0.0.1:54207_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering [junit4] 2> 770041 INFO (qtp1855766365-8317) [n:127.0.0.1:54207_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:34386_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:34386","node_name":"127.0.0.1:34386_","state":"down"} [junit4] 2> 770458 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-3-001/cores/collection1 [junit4] 2> 770459 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-3-001 [junit4] 2> 770460 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028 [junit4] 2> 770465 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@725d40ea{/,null,AVAILABLE} [junit4] 2> 770465 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@70a17786{HTTP/1.1,[http/1.1]}{127.0.0.1:40766} [junit4] 2> 770465 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.e.j.s.Server Started @773503ms [junit4] 2> 770465 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40142/hdfs__localhost_40142__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001_tempDir-002_jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=40766, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-3-001/cores} [junit4] 2> 770466 ERROR (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 770469 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0 [junit4] 2> 770469 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 770469 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 770469 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-06-10T23:21:53.107Z [junit4] 2> 770478 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 770478 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-3-001/solr.xml [junit4] 2> 770486 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 770497 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38291/solr [junit4] 2> 770519 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:40766_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3) [junit4] 2> 770529 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:40766_ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 770532 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:40766_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40766_ [junit4] 2> 770534 INFO (zkCallback-1270-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 770535 INFO (zkCallback-1275-thread-1-processing-n:127.0.0.1:54207_) [n:127.0.0.1:54207_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 770534 INFO (zkCallback-1281-thread-1-processing-n:127.0.0.1:34386_) [n:127.0.0.1:34386_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 770535 INFO (zkCallback-1266-thread-1-processing-n:127.0.0.1:60555_) [n:127.0.0.1:60555_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 770544 INFO (zkCallback-1288-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 770647 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:40766_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-3-001/cores [junit4] 2> 770647 INFO (TEST-StressHdfsTest.test-seed#[3531B02B79E0B1B4]) [n:127.0.0.1:40766_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1] [junit4] 2> 770647 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores [junit4] 2> 770649 INFO (OverseerStateUpdate-98116364258181124-127.0.0.1:60555_-n_0000000000) [n:127.0.0.1:60555_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 771041 INFO (qtp1855766365-8317) [n:127.0.0.1:54207_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:34386_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","dataDir":"hdfs://localhost:40142/solr_hdfs_home/collection1/core_node2/data/","base_url":"http://127.0.0.1:34386","node_name":"127.0.0.1:34386_","state":"recovering","ulogDir":"hdfs://localhost:40142/solr_hdfs_home/collection1/core_node2/data/tlog"} [junit4] 2> 771041 INFO (qtp1855766365-8317) [n:127.0.0.1:54207_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds. [junit4] 2> 771041 INFO (qtp1855766365-8317) [n:127.0.0.1:54207_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:34386_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001 [junit4] 2> 771542 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:54207/collection1/] - recoveringAfterStartup=[true] [junit4] 2> 771542 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:34386 START replicas=[http://127.0.0.1:54207/collection1/] nUpdates=100 [junit4] 2> 771554 INFO (qtp1855766365-8315) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0} [junit4] 2> 771554 INFO (qtp1855766365-8315) [n:127.0.0.1:54207_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=2 [junit4] 2> 771556 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0} [junit4] 2> 771556 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync [junit4] 2> 771556 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false} [junit4] 2> 771556 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit. [junit4] 2> 771557 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush [junit4] 2> 771557 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful. [junit4] 2> 771557 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync. [junit4] 2> 771557 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed. [junit4] 2> 771557 INFO (recoveryExecutor-1279-thread-1-processing-n:127.0.0.1:34386_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:34386_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery. [junit4] 2> 771670 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0 [junit4] 2> 771686 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test [junit4] 2> 771843 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id [junit4] 2> 771855 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true [junit4] 2> 771856 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40142/solr_hdfs_home [junit4] 2> 771856 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled [junit4] 2> 771856 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 771856 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001/shard-3-001/cores/collection1], dataDir=[hdfs://localhost:40142/solr_hdfs_home/collection1/core_node3/data/] [junit4] 2> 771856 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2254e2 [junit4] 2> 771857 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node3/data/snapshot_metadata [junit4] 2> 771865 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 771865 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 771865 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 771876 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 771877 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node3/data [junit4] 2> 771897 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40142/solr_hdfs_home/collection1/core_node3/data/index [junit4] 2> 771908 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true] [junit4] 2> 771908 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes [junit4] 2> 771908 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache [junit4] 2> 771930 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled [junit4] 2> 771930 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=74.70703125, floorSegmentMB=1.3876953125, forceMergeDeletesPctAllowed=12.7689070004221, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1777327330543834 [junit4] 2> 771964 INFO (IPC Server handler 3 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:54611 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7ef2b07b-693e-4c73-860f-e4e7f3a1aa3b:NORMAL:127.0.0.1:38092|RBW], ReplicaUC[[DISK]DS-308c8e3a-0af2-481b-9eeb-420d330ffeab:NORMAL:127.0.0.1:54611|FINALIZED]]} size 0 [junit4] 2> 771965 INFO (IPC Server handler 4 on 40142) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38092 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-308c8e3a-0af2-481b-9eeb-420d330ffeab:NORMAL:127.0.0.1:54611|FINALIZED], ReplicaUC[[DISK]DS-2e7827eb-ab45-4dca-b444-8b82af11d5d8:NORMAL:127.0.0.1:38092|FINALIZED]]} size 0 [junit4] 2> 771983 WARN (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}} [junit4] 2> 772061 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog [junit4] 2> 772061 INFO (coreLoadExecutor-3084-thread-1-processing-n:127.0.0.1:40766_) [n:127.0.0.1:40766_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 [...truncated too long message...] E0B1B4]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 [junit4] 2> 985459 ERROR (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal [junit4] 2> java.lang.reflect.InvocationTargetException [junit4] 2> at sun.reflect.GeneratedMethodAccessor44.invoke(Unknown Source) [junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498) [junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111) [junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144) [junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401) [junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151) [junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804) [junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595) [junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813) [junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430) [junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) [junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) [junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213) [junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594) [junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72) [junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68) [junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171) [junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872) [junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726) [junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705) [junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:203) [junit4] 2> at org.apache.solr.cloud.hdfs.StressHdfsTest.teardownClass(StressHdfsTest.java:77) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) [junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) [junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) [junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) [junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> Caused by: java.lang.NullPointerException [junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203) [junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370) [junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729) [junit4] 2> ... 53 more [junit4] 2> 985462 INFO (SUITE-StressHdfsTest-seed#[3531B02B79E0B1B4]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.StressHdfsTest_3531B02B79E0B1B4-001 [junit4] 2> Jun 10, 2017 11:25:28 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), id=FSTOrd50, text=PostingsFormat(name=Memory doPackFST= false), txt_t=PostingsFormat(name=Memory doPackFST= false)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), _version_=DocValuesFormat(name=Direct), intDefault=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Direct), intDvoDefault=DocValuesFormat(name=Lucene54), range_facet_l=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=262, maxMBSortInHeap=5.291497748953938, sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=zh-TW, timezone=Pacific/Truk [junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=2,free=47009512,total=534773760 [junit4] 2> NOTE: All tests run in this JVM: [TestSubQueryTransformer, JSONWriterTest, TestPostingsSolrHighlighter, TestTrackingShardHandlerFactory, PKIAuthenticationIntegrationTest, HdfsNNFailoverTest, DirectoryFactoryTest, PeerSyncReplicationTest, TestConfigSetsAPIZkFailure, TestJmxMonitoredMap, TestGroupingSearch, TestUninvertingReader, ActionThrottleTest, DistributedSuggestComponentTest, OverseerTest, TestObjectReleaseTracker, ConcurrentDeleteAndCreateCollectionTest, RequestHandlersTest, DeleteLastCustomShardedReplicaTest, PreAnalyzedUpdateProcessorTest, BlockJoinFacetDistribTest, SuggesterTSTTest, DeleteShardTest, BasicDistributedZk2Test, TestUseDocValuesAsStored2, TestSolrQueryParser, TestManagedSchemaAPI, TestDistributedMissingSort, TestNamedUpdateProcessors, TestHdfsCloudBackupRestore, TestUnifiedSolrHighlighter, PluginInfoTest, TestFreeTextSuggestions, MultiThreadedOCPTest, TestSort, VMParamsZkACLAndCredentialsProvidersTest, TestConfig, TestExportWriter, MigrateRouteKeyTest, V2ApiIntegrationTest, ResponseLogComponentTest, TestCollapseQParserPlugin, TimeZoneUtilsTest, TestCorePropertiesReload, AtomicUpdateProcessorFactoryTest, SolrPluginUtilsTest, BigEndianAscendingWordDeserializerTest, SolrMetricsIntegrationTest, SuggesterTest, PreAnalyzedFieldTest, TestReloadDeadlock, TestLeaderInitiatedRecoveryThread, HLLUtilTest, TestFieldResource, ShowFileRequestHandlerTest, TestCloudInspectUtil, SubstringBytesRefFilterTest, SolrIndexMetricsTest, CoreAdminHandlerTest, DistributedExpandComponentTest, CdcrReplicationDistributedZkTest, FullSolrCloudDistribCmdsTest, CircularListTest, PrimUtilsTest, TestImplicitCoreProperties, TestHdfsBackupRestoreCore, OverriddenZkACLAndCredentialsProvidersTest, SuggestComponentContextFilterQueryTest, TestSmileRequest, TestSolrDeletionPolicy1, GraphQueryTest, QueryParsingTest, StatelessScriptUpdateProcessorFactoryTest, PingRequestHandlerTest, StressHdfsTest] [junit4] Completed [172/715 (1!)] on J2 in 234.58s, 1 test, 1 error <<< FAILURES! [...truncated 45162 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
