Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/18363/
Java: 64bit/jdk-9-ea+140 -XX:-UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test
Error Message:
There are still nodes recoverying - waited for 30 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 30
seconds
at
__randomizedtesting.SeedInfo.seed([8132DF4AE0C61408:966E0904E3A79F0]:0)
at org.junit.Assert.fail(Assert.java:93)
at
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:184)
at
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:862)
at
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1418)
at
org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:136)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native
Method)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:535)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Build Log:
[...truncated 10781 lines...]
[junit4] Suite: org.apache.solr.cloud.LeaderFailureAfterFreshStartTest
[junit4] 2> Creating dataDir:
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/init-core-data-001
[junit4] 2> 49992 INFO
(SUITE-LeaderFailureAfterFreshStartTest-seed#[8132DF4AE0C61408]-worker) [ ]
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via:
@org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0,
clientAuth=0.0/0.0)
[junit4] 2> 49994 INFO
(SUITE-LeaderFailureAfterFreshStartTest-seed#[8132DF4AE0C61408]-worker) [ ]
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_sz/fb
[junit4] 2> 49997 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 50015 INFO (Thread-82) [ ] o.a.s.c.ZkTestServer client
port:0.0.0.0/0.0.0.0:0
[junit4] 2> 50015 INFO (Thread-82) [ ] o.a.s.c.ZkTestServer Starting
server
[junit4] 2> 50115 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkTestServer start zk server on port:46183
[junit4] 2> 50141 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
to /configs/conf1/solrconfig.xml
[junit4] 2> 50146 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml
to /configs/conf1/schema.xml
[junit4] 2> 50151 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 50156 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
to /configs/conf1/stopwords.txt
[junit4] 2> 50162 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt
to /configs/conf1/protwords.txt
[junit4] 2> 50167 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml
to /configs/conf1/currency.xml
[junit4] 2> 50172 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
to /configs/conf1/enumsConfig.xml
[junit4] 2> 50176 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
to /configs/conf1/open-exchange-rates.json
[junit4] 2> 50181 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 50186 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
to /configs/conf1/old_synonyms.txt
[junit4] 2> 50190 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractZkTestCase put
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
to /configs/conf1/synonyms.txt
[junit4] 2> 51153 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.SolrTestCaseJ4 Writing core.properties file to
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/control-001/cores/collection1
[junit4] 2> 51163 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server jetty-9.3.8.v20160314
[junit4] 2> 51179 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Started
o.e.j.s.ServletContextHandler@6de203ff{/_sz/fb,null,AVAILABLE}
[junit4] 2> 51181 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Started ServerConnector@7507bdba{SSL,[ssl,
http/1.1]}{127.0.0.1:46541}
[junit4] 2> 51181 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server Started @53953ms
[junit4] 2> 51181 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/_sz/fb,
solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/tempDir-001/control/data,
hostPort=46541,
coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/control-001/cores}
[junit4] 2> 51182 ERROR
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be
missing or incomplete.
[junit4] 2> 51189 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version
7.0.0
[junit4] 2> 51189 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 51189 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 51189 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
2016-11-24T16:59:42.771461Z
[junit4] 2> 51196 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in
ZooKeeper)
[junit4] 2> 51196 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.SolrXmlConfig Loading container configuration from
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/control-001/solr.xml
[junit4] 2> 51208 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46183/solr
[junit4] 2> 51252 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.OverseerElectionContext I am going to be the leader
127.0.0.1:46541__sz%2Ffb
[junit4] 2> 51254 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.Overseer Overseer
(id=96993724439527429-127.0.0.1:46541__sz%2Ffb-n_0000000000) starting
[junit4] 2> 51262 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkController Register node as live in
ZooKeeper:/live_nodes/127.0.0.1:46541__sz%2Ffb
[junit4] 2> 51272 INFO (zkCallback-99-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 51317 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/control-001/cores
[junit4] 2> 51317 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 51350 INFO
(OverseerStateUpdate-96993724439527429-127.0.0.1:46541__sz%2Ffb-n_0000000000) [
] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 51372 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 51382 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 51474 WARN (coreLoadExecutor-453-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] default search field in schema is text.
WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 51476 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 51496 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from
collection control_collection
[junit4] 2> 51500 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at
[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/control-001/cores/collection1],
dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/control-001/cores/collection1/data/]
[junit4] 2> 51500 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@339beff
[junit4] 2> 51506 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=39, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=29.8193359375,
floorSegmentMB=1.9169921875, forceMergeDeletesPctAllowed=17.059931898930447,
segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.6773153240973162
[junit4] 2> 51538 WARN (coreLoadExecutor-453-thread-1) [ ]
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type =
requestHandler,name = /dump,class = DumpRequestHandler,args =
{defaults={a=A,b=B}}}
[junit4] 2> 51559 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.UpdateHandler Using UpdateLog implementation:
org.apache.solr.update.UpdateLog
[junit4] 2> 51559 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 51561 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 51561 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 51564 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=45, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=65.869140625,
floorSegmentMB=0.5126953125, forceMergeDeletesPctAllowed=9.226548773194736,
segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.48499351052231954
[junit4] 2> 51566 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.s.SolrIndexSearcher Opening [Searcher@1cb8d279[collection1] main]
[junit4] 2> 51567 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase:
/configs/conf1
[junit4] 2> 51568 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using
ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 51568 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 51569 INFO (searcherExecutor-454-thread-1) [ ]
o.a.s.c.SolrCore [collection1] Registered new searcher
Searcher@1cb8d279[collection1]
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 51569 INFO (coreLoadExecutor-453-thread-1) [ ]
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using
new clock 1551899592649342976
[junit4] 2> 51579 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 51580 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 51580 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.SyncStrategy Sync replicas to
https://127.0.0.1:46541/_sz/fb/collection1/
[junit4] 2> 51581 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 51581 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.SyncStrategy https://127.0.0.1:46541/_sz/fb/collection1/ has no replicas
[junit4] 2> 51590 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.ShardLeaderElectionContext I am the new leader:
https://127.0.0.1:46541/_sz/fb/collection1/ shard1
[junit4] 2> 51742 INFO (coreZkRegister-446-thread-1) [ ]
o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 51903 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 51906 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection
loss:false
[junit4] 2> 51907 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 52840 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.SolrTestCaseJ4 Writing core.properties file to
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001/cores/collection1
[junit4] 2> 52841 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001
[junit4] 2> 52845 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server jetty-9.3.8.v20160314
[junit4] 2> 52846 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Started
o.e.j.s.ServletContextHandler@37d48eca{/_sz/fb,null,AVAILABLE}
[junit4] 2> 52848 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Started ServerConnector@ed2b5da{SSL,[ssl,
http/1.1]}{127.0.0.1:36344}
[junit4] 2> 52848 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server Started @55619ms
[junit4] 2> 52848 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/_sz/fb,
solrconfig=solrconfig.xml,
solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/tempDir-001/jetty1,
hostPort=36344,
coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001/cores}
[junit4] 2> 52848 ERROR
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be
missing or incomplete.
[junit4] 2> 52849 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version
7.0.0
[junit4] 2> 52849 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 52849 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 52849 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
2016-11-24T16:59:44.431319Z
[junit4] 2> 52857 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in
ZooKeeper)
[junit4] 2> 52857 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.SolrXmlConfig Loading container configuration from
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001/solr.xml
[junit4] 2> 52866 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46183/solr
[junit4] 2> 52875 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 52879 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkController Register node as live in
ZooKeeper:/live_nodes/127.0.0.1:36344__sz%2Ffb
[junit4] 2> 52881 INFO (zkCallback-103-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 52884 INFO (zkCallback-99-thread-3) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 52887 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 52949 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001/cores
[junit4] 2> 52949 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 52954 INFO
(OverseerStateUpdate-96993724439527429-127.0.0.1:46541__sz%2Ffb-n_0000000000) [
] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 53063 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [2])
[junit4] 2> 53965 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 53979 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 54080 WARN (coreLoadExecutor-464-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] default search field in schema is text.
WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 54084 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 54104 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from
collection collection1
[junit4] 2> 54105 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at
[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001/cores/collection1],
dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 54105 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@339beff
[junit4] 2> 54111 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=39, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=29.8193359375,
floorSegmentMB=1.9169921875, forceMergeDeletesPctAllowed=17.059931898930447,
segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.6773153240973162
[junit4] 2> 54156 WARN (coreLoadExecutor-464-thread-1) [ ]
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type =
requestHandler,name = /dump,class = DumpRequestHandler,args =
{defaults={a=A,b=B}}}
[junit4] 2> 54164 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.UpdateHandler Using UpdateLog implementation:
org.apache.solr.update.UpdateLog
[junit4] 2> 54164 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 54165 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 54165 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 54166 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=45, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=65.869140625,
floorSegmentMB=0.5126953125, forceMergeDeletesPctAllowed=9.226548773194736,
segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.48499351052231954
[junit4] 2> 54168 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.s.SolrIndexSearcher Opening [Searcher@4c2b1cfa[collection1] main]
[junit4] 2> 54169 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase:
/configs/conf1
[junit4] 2> 54169 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using
ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 54170 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 54171 INFO (searcherExecutor-465-thread-1) [ ]
o.a.s.c.SolrCore [collection1] Registered new searcher
Searcher@4c2b1cfa[collection1]
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 54171 INFO (coreLoadExecutor-464-thread-1) [ ]
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using
new clock 1551899595377737728
[junit4] 2> 54178 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 54178 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 54178 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.SyncStrategy Sync replicas to
https://127.0.0.1:36344/_sz/fb/collection1/
[junit4] 2> 54178 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 54178 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.SyncStrategy https://127.0.0.1:36344/_sz/fb/collection1/ has no replicas
[junit4] 2> 54186 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.ShardLeaderElectionContext I am the new leader:
https://127.0.0.1:36344/_sz/fb/collection1/ shard1
[junit4] 2> 54289 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [2])
[junit4] 2> 54338 INFO (coreZkRegister-459-thread-1) [ ]
o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 54441 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [2])
[junit4] 2> 55127 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.SolrTestCaseJ4 Writing core.properties file to
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/cores/collection1
[junit4] 2> 55128 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001
[junit4] 2> 55131 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server jetty-9.3.8.v20160314
[junit4] 2> 55132 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Started
o.e.j.s.ServletContextHandler@7f54daa8{/_sz/fb,null,AVAILABLE}
[junit4] 2> 55133 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Started ServerConnector@79a70f7f{SSL,[ssl,
http/1.1]}{127.0.0.1:35589}
[junit4] 2> 55133 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server Started @57904ms
[junit4] 2> 55133 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/_sz/fb,
solrconfig=solrconfig.xml,
solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/tempDir-001/jetty2,
hostPort=35589,
coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/cores}
[junit4] 2> 55133 ERROR
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be
missing or incomplete.
[junit4] 2> 55134 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version
7.0.0
[junit4] 2> 55134 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 55134 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 55134 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
2016-11-24T16:59:46.716169Z
[junit4] 2> 55137 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in
ZooKeeper)
[junit4] 2> 55137 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.SolrXmlConfig Loading container configuration from
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/solr.xml
[junit4] 2> 55148 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46183/solr
[junit4] 2> 55167 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 55171 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkController Register node as live in
ZooKeeper:/live_nodes/127.0.0.1:35589__sz%2Ffb
[junit4] 2> 55172 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 55172 INFO (zkCallback-99-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 55172 INFO (zkCallback-103-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 55179 INFO (zkCallback-115-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 55197 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/cores
[junit4] 2> 55197 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 55207 INFO
(OverseerStateUpdate-96993724439527429-127.0.0.1:46541__sz%2Ffb-n_0000000000) [
] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 55310 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 55310 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 56220 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 56251 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 56373 WARN (coreLoadExecutor-475-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] default search field in schema is text.
WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 56376 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 56392 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from
collection collection1
[junit4] 2> 56392 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at
[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/cores/collection1],
dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 56392 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@339beff
[junit4] 2> 56398 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=39, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=29.8193359375,
floorSegmentMB=1.9169921875, forceMergeDeletesPctAllowed=17.059931898930447,
segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.6773153240973162
[junit4] 2> 56463 WARN (coreLoadExecutor-475-thread-1) [ ]
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type =
requestHandler,name = /dump,class = DumpRequestHandler,args =
{defaults={a=A,b=B}}}
[junit4] 2> 56479 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.UpdateHandler Using UpdateLog implementation:
org.apache.solr.update.UpdateLog
[junit4] 2> 56479 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 56480 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 56480 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 56482 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=45, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=65.869140625,
floorSegmentMB=0.5126953125, forceMergeDeletesPctAllowed=9.226548773194736,
segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.48499351052231954
[junit4] 2> 56484 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.s.SolrIndexSearcher Opening [Searcher@5f93d591[collection1] main]
[junit4] 2> 56485 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase:
/configs/conf1
[junit4] 2> 56486 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using
ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 56486 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 56487 INFO (searcherExecutor-476-thread-1) [ ]
o.a.s.c.SolrCore [collection1] Registered new searcher
Searcher@5f93d591[collection1]
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 56487 INFO (coreLoadExecutor-475-thread-1) [ ]
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using
new clock 1551899597806239744
[junit4] 2> 56492 INFO (coreZkRegister-470-thread-1) [ ]
o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 56493 INFO (updateExecutor-112-thread-1) [ ]
o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 56496 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 56497 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 56497 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 56497 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE,
tlog=null}
[junit4] 2> 56497 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering,
leader is [https://127.0.0.1:36344/_sz/fb/collection1/] and I am
[https://127.0.0.1:35589/_sz/fb/collection1/]
[junit4] 2> 56501 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Sending prep recovery command to
[https://127.0.0.1:36344/_sz/fb]; [WaitForState:
action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:35589__sz%252Ffb&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 56557 INFO (qtp1756983472-595) [ ]
o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state:
recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 56558 INFO (qtp1756983472-595) [ ]
o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1
(shard1 of collection1) have state: recovering
[junit4] 2> 56558 INFO (qtp1756983472-595) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=true, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:35589__sz%2Ffb, coreNodeName=core_node2,
onlyIfActiveCheckResult=false, nodeProps:
core_node2:{"core":"collection1","base_url":"https://127.0.0.1:35589/_sz/fb","node_name":"127.0.0.1:35589__sz%2Ffb","state":"down"}
[junit4] 2> 56604 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 56604 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 57558 INFO (qtp1756983472-595) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=true, checkLive=true, currentState=recovering, localState=active,
nodeName=127.0.0.1:35589__sz%2Ffb, coreNodeName=core_node2,
onlyIfActiveCheckResult=false, nodeProps:
core_node2:{"core":"collection1","base_url":"https://127.0.0.1:35589/_sz/fb","node_name":"127.0.0.1:35589__sz%2Ffb","state":"recovering"}
[junit4] 2> 57558 INFO (qtp1756983472-595) [ ]
o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering,
checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 57558 INFO (qtp1756983472-595) [ ] o.a.s.s.HttpSolrCall
[admin] webapp=null path=/admin/cores
params={nodeName=127.0.0.1:35589__sz%252Ffb&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2}
status=0 QTime=1001
[junit4] 2> 57606 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.SolrTestCaseJ4 Writing core.properties file to
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001/cores/collection1
[junit4] 2> 57607 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001
[junit4] 2> 57610 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server jetty-9.3.8.v20160314
[junit4] 2> 57615 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Started
o.e.j.s.ServletContextHandler@6ff03606{/_sz/fb,null,AVAILABLE}
[junit4] 2> 57616 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Started ServerConnector@3647aae1{SSL,[ssl,
http/1.1]}{127.0.0.1:44454}
[junit4] 2> 57616 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.Server Started @60388ms
[junit4] 2> 57617 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/_sz/fb,
solrconfig=solrconfig.xml,
solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/tempDir-001/jetty3,
hostPort=44454,
coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001/cores}
[junit4] 2> 57617 ERROR
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be
missing or incomplete.
[junit4] 2> 57617 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version
7.0.0
[junit4] 2> 57617 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 57617 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 57617 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
2016-11-24T16:59:49.199605Z
[junit4] 2> 57621 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in
ZooKeeper)
[junit4] 2> 57621 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.SolrXmlConfig Loading container configuration from
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001/solr.xml
[junit4] 2> 57633 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46183/solr
[junit4] 2> 57647 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 57651 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkController Register node as live in
ZooKeeper:/live_nodes/127.0.0.1:44454__sz%2Ffb
[junit4] 2> 57657 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 57657 INFO (zkCallback-99-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 57657 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 57657 INFO (zkCallback-103-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 57659 INFO (zkCallback-122-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 57706 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001/cores
[junit4] 2> 57706 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 57710 INFO
(OverseerStateUpdate-96993724439527429-127.0.0.1:46541__sz%2Ffb-n_0000000000) [
] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 57812 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [4])
[junit4] 2> 57812 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [4])
[junit4] 2> 57812 INFO (zkCallback-122-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [4])
[junit4] 2> 58753 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 58769 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 58868 WARN (coreLoadExecutor-486-thread-1) [ ]
o.a.s.s.IndexSchema [collection1] default search field in schema is text.
WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 58871 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 58880 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from
collection collection1
[junit4] 2> 58881 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at
[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001/cores/collection1],
dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 58881 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@339beff
[junit4] 2> 58886 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=39, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=29.8193359375,
floorSegmentMB=1.9169921875, forceMergeDeletesPctAllowed=17.059931898930447,
segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.6773153240973162
[junit4] 2> 58917 WARN (coreLoadExecutor-486-thread-1) [ ]
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type =
requestHandler,name = /dump,class = DumpRequestHandler,args =
{defaults={a=A,b=B}}}
[junit4] 2> 58926 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.UpdateHandler Using UpdateLog implementation:
org.apache.solr.update.UpdateLog
[junit4] 2> 58926 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 58927 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 58927 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 58928 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=45, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=65.869140625,
floorSegmentMB=0.5126953125, forceMergeDeletesPctAllowed=9.226548773194736,
segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=0.48499351052231954
[junit4] 2> 58929 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.s.SolrIndexSearcher Opening [Searcher@299f3b33[collection1] main]
[junit4] 2> 58930 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase:
/configs/conf1
[junit4] 2> 58930 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using
ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 58930 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 58931 INFO (searcherExecutor-487-thread-1) [ ]
o.a.s.c.SolrCore [collection1] Registered new searcher
Searcher@299f3b33[collection1]
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 58932 INFO (coreLoadExecutor-486-thread-1) [ ]
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using
new clock 1551899600370008064
[junit4] 2> 58936 INFO (coreZkRegister-481-thread-1) [ ]
o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 58936 INFO (updateExecutor-119-thread-1) [ ]
o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 58936 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 58937 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 58937 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 58937 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE,
tlog=null}
[junit4] 2> 58937 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering,
leader is [https://127.0.0.1:36344/_sz/fb/collection1/] and I am
[https://127.0.0.1:44454/_sz/fb/collection1/]
[junit4] 2> 58940 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Sending prep recovery command to
[https://127.0.0.1:36344/_sz/fb]; [WaitForState:
action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:44454__sz%252Ffb&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 58948 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state:
recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 58949 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1
(shard1 of collection1) have state: recovering
[junit4] 2> 58949 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=true, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 59040 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [4])
[junit4] 2> 59040 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [4])
[junit4] 2> 59040 INFO (zkCallback-122-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [4])
[junit4] 2> 59213 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 59213 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.LeaderFailureAfterFreshStartTest Leader node_name: core_node1, url:
https://127.0.0.1:36344/_sz/fb/collection1
[junit4] 2> 59213 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.LeaderFailureAfterFreshStartTest Nonleader node_name: core_node2, url:
https://127.0.0.1:35589/_sz/fb/collection1
[junit4] 2> 59214 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.LeaderFailureAfterFreshStartTest Nonleader node_name: core_node3, url:
https://127.0.0.1:44454/_sz/fb/collection1
[junit4] 2> 59214 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ChaosMonkey monkey: kill shard! 44454
[junit4] 2> 59215 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Stopped ServerConnector@3647aae1{SSL,[ssl,
http/1.1]}{127.0.0.1:0}
[junit4] 2> 59215 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CoreContainer Shutting down CoreContainer instance=1226165034
[junit4] 2> 59217 WARN
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1]
coreNodeName=[core_node3]
[junit4] 2> 59218 ERROR (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Error while trying to recover.
core=collection1:java.util.concurrent.ExecutionException:
org.apache.solr.client.solrj.SolrServerException: IOException occured when
talking to server at: https://127.0.0.1:36344/_sz/fb
[junit4] 2> at
java.util.concurrent.FutureTask.report(java.base@9-ea/FutureTask.java:122)
[junit4] 2> at
java.util.concurrent.FutureTask.get(java.base@9-ea/FutureTask.java:191)
[junit4] 2> at
org.apache.solr.cloud.RecoveryStrategy.sendPrepRecoveryCmd(RecoveryStrategy.java:614)
[junit4] 2> at
org.apache.solr.cloud.RecoveryStrategy.sendPrepRecoveryCmd(RecoveryStrategy.java:591)
[junit4] 2> at
org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:351)
[junit4] 2> at
org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:222)
[junit4] 2> at
java.util.concurrent.Executors$RunnableAdapter.call(java.base@9-ea/Executors.java:514)
[junit4] 2> at
java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:264)
[junit4] 2> at
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1161)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:635)
[junit4] 2> at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
[junit4] 2> Caused by: org.apache.solr.client.solrj.SolrServerException:
IOException occured when talking to server at: https://127.0.0.1:36344/_sz/fb
[junit4] 2> at
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:624)
[junit4] 2> at
org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:302)
[junit4] 2> ... 5 more
[junit4] 2> Caused by: java.net.SocketException: Socket closed
[junit4] 2> at
java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:203)
[junit4] 2> at
java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
[junit4] 2> at
sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
[junit4] 2> at
sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
[junit4] 2> at
sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:913)
[junit4] 2> at
sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
[junit4] 2> at
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
[junit4] 2> at
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
[junit4] 2> at
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
[junit4] 2> at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
[junit4] 2> at
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
[junit4] 2> at
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
[junit4] 2> at
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
[junit4] 2> at
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
[junit4] 2> at
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
[junit4] 2> at
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
[junit4] 2> at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:515)
[junit4] 2> ... 6 more
[junit4] 2>
[junit4] 2> 59218 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Replay not started, or was not successful... still
buffering updates.
[junit4] 2> 59218 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 59218 INFO (recoveryExecutor-120-thread-1) [ ]
o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 59234 INFO (coreCloseExecutor-492-thread-1) [ ]
o.a.s.c.SolrCore [collection1] CLOSING SolrCore
org.apache.solr.core.SolrCore@7926b453
[junit4] 2> 59235 WARN (coreCloseExecutor-492-thread-1) [ ]
o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1]
coreNodeName=[core_node3]
[junit4] 2> 59248 WARN (zkCallback-122-thread-1) [ ]
o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK:
[KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 59249 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 59249 INFO (zkCallback-103-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 59249 INFO (zkCallback-99-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 59249 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 59249 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Stopped
o.e.j.s.ServletContextHandler@6ff03606{/_sz/fb,null,UNAVAILABLE}
[junit4] 2> 59250 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ChaosMonkey monkey: stop shard! 44454
[junit4] 2> 59250 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase Wait for no shard inconsistency
[junit4] 2> 59270 INFO (qtp307103912-557) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 59271 INFO (qtp307103912-557) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 59271 INFO (qtp307103912-557) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 59272 INFO (qtp307103912-557) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
0 3
[junit4] 2> 59312 INFO (qtp1756983472-594) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 59313 INFO (qtp1756983472-594) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 59313 INFO (qtp1756983472-594) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 59314 INFO (qtp1756983472-594) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
0 2
[junit4] 2> 59321 INFO (qtp1993295353-628) [ ]
o.a.s.u.p.DistributedUpdateProcessor Ignoring commit while not ACTIVE - state:
BUFFERING replay: false
[junit4] 2> 59321 INFO (qtp1993295353-628) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
0 0
[junit4] 2> 59323 INFO (qtp1756983472-595) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
0 26
[junit4] 2> check const of shard1
[junit4] 2> client0
[junit4] 2>
PROPS:core_node1:{"core":"collection1","base_url":"https://127.0.0.1:36344/_sz/fb","node_name":"127.0.0.1:36344__sz%2Ffb","state":"active","leader":"true"}
[junit4] 2> 59332 INFO (qtp1756983472-594) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/select
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
hits=0 status=0 QTime=1
[junit4] 2> live:true
[junit4] 2> num:0
[junit4] 2>
[junit4] 2> client1
[junit4] 2>
PROPS:core_node2:{"core":"collection1","base_url":"https://127.0.0.1:35589/_sz/fb","node_name":"127.0.0.1:35589__sz%2Ffb","state":"recovering"}
[junit4] 2> 59341 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 59341 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 59347 INFO (qtp1993295353-627) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/select
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
hits=0 status=0 QTime=0
[junit4] 2> live:true
[junit4] 2> num:0
[junit4] 2>
[junit4] 2> client2
[junit4] 2>
PROPS:core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"recovering"}
[junit4] 2> error contacting client: Server refused connection at:
https://127.0.0.1:44454/_sz/fb/collection1
[junit4] 2>
[junit4] 2> 59949 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 60949 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 61366 INFO (qtp307103912-558) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={wt=javabin&version=2}{deleteByQuery=*:* (-1551899602907561984)} 0 14
[junit4] 2> 61377 INFO (qtp1993295353-621) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&_version_=-1551899602925387776&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{deleteByQuery=*:*
(-1551899602925387776)} 0 1
[junit4] 2> 61390 INFO (qtp1756983472-593) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={wt=javabin&version=2}{deleteByQuery=*:* (-1551899602925387776)} 0 20
[junit4] 2> 61391 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30
for each attempt
[junit4] 2> 61391 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection:
collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 61949 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 62950 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 63950 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 64561 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Attempting to PeerSync from
[https://127.0.0.1:36344/_sz/fb/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 64561 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:35589/_sz/fb
START replicas=[https://127.0.0.1:36344/_sz/fb/collection1/] nUpdates=1000
[junit4] 2> 64572 INFO (qtp1756983472-594) [ ]
o.a.s.u.IndexFingerprint IndexFingerprint millis:4.0
result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0,
maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 64572 INFO (qtp1756983472-594) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/get
params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2}
status=0 QTime=5
[junit4] 2> 64574 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0
result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0,
maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 64574 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 64575 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 64575 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 64575 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 64575 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 64575 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 64577 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Replaying buffered documents.
[junit4] 2> 64577 WARN (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.UpdateLog Starting log replay
tlog{file=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001/shard-2-001/cores/collection1/data/tlog/tlog.0000000000000000000
refcount=2} active=true starting pos=0
[junit4] 2> 64581 INFO (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{flags=2,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 64581 INFO (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 64582 INFO (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 64582 INFO (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] {deleteByQuery=*:*
(-1551899602925387776)} 0 4
[junit4] 2> 64582 INFO (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.UpdateLog Re-computing max version from index after log re-play.
[junit4] 2> 64585 WARN (recoveryExecutor-478-thread-1) [ ]
o.a.s.u.UpdateLog Log replay finished. recoveryInfo=RecoveryInfo{adds=0
deletes=0 deleteByQuery=1 errors=0 positionOfStart=0}
[junit4] 2> 64585 INFO (recoveryExecutor-113-thread-1) [ ]
o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 64587 INFO (zkCallback-109-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 64587 INFO (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [3])
[junit4] 2> 64950 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 65401 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 65405 INFO (qtp307103912-559) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65405 INFO (qtp307103912-559) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 65407 INFO (qtp307103912-559) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65407 INFO (qtp307103912-559) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
0 2
[junit4] 2> 65417 INFO (qtp1993295353-622) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65417 INFO (qtp1993295353-622) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 65418 INFO (qtp1993295353-622) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65418 INFO (qtp1993295353-622) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
0 1
[junit4] 2> 65420 INFO (qtp1756983472-697) [ ]
o.a.s.u.DirectUpdateHandler2 start
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65421 INFO (qtp1756983472-697) [ ]
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 65422 INFO (qtp1756983472-697) [ ]
o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65422 INFO (qtp1756983472-697) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
0 1
[junit4] 2> 65424 INFO (qtp1756983472-598) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
0 12
[junit4] 2> 65429 INFO (qtp1756983472-595) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/select
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
hits=0 status=0 QTime=0
[junit4] 2> 65433 INFO (qtp1993295353-628) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/select
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
hits=0 status=0 QTime=0
[junit4] 2> 65952 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> 66952 INFO (qtp1756983472-597) [ ]
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1,
shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader?
true, live=false, checkLive=true, currentState=down, localState=active,
nodeName=127.0.0.1:44454__sz%2Ffb, coreNodeName=core_node3,
onlyIfActiveCheckResult=false, nodeProps:
core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> check const of shard1
[junit4] 2> client0
[junit4] 2>
PROPS:core_node1:{"core":"collection1","base_url":"https://127.0.0.1:36344/_sz/fb","node_name":"127.0.0.1:36344__sz%2Ffb","state":"active","leader":"true"}
[junit4] 2> 67440 INFO (qtp1756983472-594) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/select
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
hits=0 status=0 QTime=1
[junit4] 2> live:true
[junit4] 2> num:0
[junit4] 2>
[junit4] 2> client1
[junit4] 2>
PROPS:core_node2:{"core":"collection1","base_url":"https://127.0.0.1:35589/_sz/fb","node_name":"127.0.0.1:35589__sz%2Ffb","state":"active"}
[junit4] 2> 67442 INFO (qtp1993295353-627) [ ] o.a.s.c.S.Request
[collection1] webapp=/_sz/fb path=/select
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
hits=0 status=0 QTime=0
[junit4] 2> live:true
[junit4] 2> num:0
[junit4] 2>
[junit4] 2> client2
[junit4] 2>
PROPS:core_node3:{"core":"collection1","base_url":"https://127.0.0.1:44454/_sz/fb","node_name":"127.0.0.1:44454__sz%2Ffb","state":"down"}
[junit4] 2> error contacting client: Server refused connection at:
https://127.0.0.1:44454/_sz/fb/collection1
[junit4] 2>
[junit4] 2> 67456 INFO (qtp1993295353-626) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[0
(1551899609298632704)]} 0 3
[junit4] 2> 67456 INFO (qtp1756983472-696) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[0
(1551899609298632704)]} 0 9
[junit4] 2> 67465 INFO (qtp1993295353-622) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[1
(1551899609311215616)]} 0 3
[junit4] 2> 67466 INFO (qtp1756983472-598) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[1
(1551899609311215616)]} 0 7
[junit4] 2> 67472 INFO (qtp1993295353-628) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[2
(1551899609321701376)]} 0 1
[junit4] 2> 67473 INFO (qtp1756983472-595) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[2
(1551899609321701376)]} 0 4
[junit4] 2> 67479 INFO (qtp1993295353-627) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[3
(1551899609329041408)]} 0 1
[junit4] 2> 67480 INFO (qtp1756983472-594) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[3
(1551899609329041408)]} 0 4
[junit4] 2> 67485 INFO (qtp1993295353-626) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[4
(1551899609335332864)]} 0 1
[junit4] 2> 67486 INFO (qtp1756983472-696) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[4
(1551899609335332864)]} 0 4
[junit4] 2> 67492 INFO (qtp1993295353-622) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[5
(1551899609342672896)]} 0 1
[junit4] 2> 67493 INFO (qtp1756983472-598) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[5
(1551899609342672896)]} 0 3
[junit4] 2> 67497 INFO (qtp1993295353-628) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[6
(1551899609348964352)]} 0 0
[junit4] 2> 67498 INFO (qtp1756983472-595) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[6
(1551899609348964352)]} 0 3
[junit4] 2> 67504 INFO (qtp1993295353-627) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[7
(1551899609355255808)]} 0 0
[junit4] 2> 67504 INFO (qtp1756983472-594) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[7
(1551899609355255808)]} 0 3
[junit4] 2> 67509 INFO (qtp1993295353-626) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[8
(1551899609361547264)]} 0 0
[junit4] 2> 67509 INFO (qtp1756983472-696) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[8
(1551899609361547264)]} 0 3
[junit4] 2> 67515 INFO (qtp1993295353-622) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[9
(1551899609366790144)]} 0 1
[junit4] 2> 67516 INFO (qtp1756983472-598) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[9
(1551899609366790144)]} 0 4
[junit4] 2> 67522 INFO (qtp1993295353-628) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36344/_sz/fb/collection1/&wt=javabin&version=2}{add=[10
(1551899609374130176)]} 0 1
[junit4] 2> 67524 INFO (qtp1756983472-595) [ ]
o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/_sz/fb path=/update
params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[
[...truncated too long message...]
tions/collection1/state.json] for collection [collection1] has occurred -
updating... (live nodes size: [1])
[junit4] 2> 106715 WARN
(OverseerStateUpdate-96993724439527443-127.0.0.1:44454__sz%2Ffb-n_0000000004) [
] o.a.s.c.Overseer Bad version writing to ZK using compare-and-set, will
force refresh cluster state: KeeperErrorCode = BadVersion for
/collections/collection1/state.json
[junit4] 2> 106718 INFO (zkCallback-129-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [1])
[junit4] 2> 106720 INFO (zkCallback-129-thread-1) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [1])
[junit4] 2> 108101 WARN (zkCallback-115-thread-2) [ ]
o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK:
[KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 109096 WARN (zkCallback-115-thread-3) [ ]
o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 109097 INFO (zkCallback-115-thread-3) [ ] o.a.s.c.SolrCore
[collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@2b68f0d
[junit4] 2> 109102 INFO (zkCallback-129-thread-3) [ ]
o.a.s.c.SyncStrategy Sync replicas to
https://127.0.0.1:44454/_sz/fb/collection1/
[junit4] 2> 109102 INFO (zkCallback-129-thread-3) [ ]
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 109102 INFO (zkCallback-129-thread-3) [ ]
o.a.s.c.SyncStrategy https://127.0.0.1:44454/_sz/fb/collection1/ has no replicas
[junit4] 2> 109105 INFO (zkCallback-129-thread-3) [ ]
o.a.s.c.ShardLeaderElectionContext I am the new leader:
https://127.0.0.1:44454/_sz/fb/collection1/ shard1
[junit4] 2> 109105 INFO (zkCallback-129-thread-3) [ ]
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent
state:SyncConnected type:NodeDataChanged
path:/collections/collection1/state.json] for collection [collection1] has
occurred - updating... (live nodes size: [1])
[junit4] 2> 109118 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Stopped ServerConnector@79a70f7f{SSL,[ssl,
http/1.1]}{127.0.0.1:0}
[junit4] 2> 109118 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Stopped
o.e.j.s.ServletContextHandler@7f54daa8{/_sz/fb,null,UNAVAILABLE}
[junit4] 2> 109118 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ChaosMonkey monkey: stop shard! 44454
[junit4] 2> 109118 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.CoreContainer Shutting down CoreContainer instance=227334388
[junit4] 2> 109120 INFO (coreCloseExecutor-518-thread-1) [ ]
o.a.s.c.SolrCore [collection1] CLOSING SolrCore
org.apache.solr.core.SolrCore@5defefa7
[junit4] 2> 109155 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.Overseer Overseer
(id=96993724439527443-127.0.0.1:44454__sz%2Ffb-n_0000000004) closing
[junit4] 2> 109156 INFO
(OverseerStateUpdate-96993724439527443-127.0.0.1:44454__sz%2Ffb-n_0000000004) [
] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44454__sz%2Ffb
[junit4] 2> 109156 WARN (zkCallback-129-thread-2) [ ]
o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK:
[KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 109157 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.ServerConnector Stopped ServerConnector@50929cd4{SSL,[ssl,
http/1.1]}{127.0.0.1:44454}
[junit4] 2> 109157 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.e.j.s.h.ContextHandler Stopped
o.e.j.s.ServletContextHandler@d2444b1{/_sz/fb,null,UNAVAILABLE}
[junit4] 2> 109158 INFO
(TEST-LeaderFailureAfterFreshStartTest.test-seed#[8132DF4AE0C61408]) [ ]
o.a.s.c.ZkTestServer connecting to 127.0.0.1:46183 46183
[junit4] 2> 109269 INFO (Thread-82) [ ] o.a.s.c.ZkTestServer
connecting to 127.0.0.1:46183 46183
[junit4] 2> 109269 WARN (Thread-82) [ ] o.a.s.c.ZkTestServer Watch
limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2
/solr/overseer_elect/election/96993724439527437-127.0.0.1:35589__sz%2Ffb-n_0000000002
[junit4] 2> 2
/solr/collections/collection1/leader_elect/shard1/election/96993724439527437-core_node2-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 57 /solr/overseer/collection-queue-work
[junit4] 2> 26 /solr/overseer/queue
[junit4] 2> 13 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test
-Dtestcase=LeaderFailureAfterFreshStartTest -Dtests.method=test
-Dtests.seed=8132DF4AE0C61408 -Dtests.multiplier=3 -Dtests.slow=true
-Dtests.locale=mer -Dtests.timezone=Africa/Conakry -Dtests.asserts=true
-Dtests.file.encoding=US-ASCII
[junit4] FAILURE 59.3s J2 | LeaderFailureAfterFreshStartTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: There are still nodes
recoverying - waited for 30 seconds
[junit4] > at
__randomizedtesting.SeedInfo.seed([8132DF4AE0C61408:966E0904E3A79F0]:0)
[junit4] > at
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:184)
[junit4] > at
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:862)
[junit4] > at
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1418)
[junit4] > at
org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:136)
[junit4] > at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native
Method)
[junit4] > at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
[junit4] > at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
[junit4] > at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
[junit4] 2> 109271 INFO
(SUITE-LeaderFailureAfterFreshStartTest-seed#[8132DF4AE0C61408]-worker) [ ]
o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at:
/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailureAfterFreshStartTest_8132DF4AE0C61408-001
[junit4] 2> Nov 24, 2016 5:00:40 PM
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked
thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70):
{other_tl1=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
range_facet_l_dv=BlockTreeOrds(blocksize=128),
rnd_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval),
intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
a_i1=PostingsFormat(name=LuceneVarGapFixedInterval),
range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval),
_version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
a_t=PostingsFormat(name=LuceneVarGapFixedInterval),
id=BlockTreeOrds(blocksize=128),
range_facet_i_dv=PostingsFormat(name=LuceneVarGapFixedInterval), text=FST50,
timestamp=PostingsFormat(name=LuceneVarGapFixedInterval)},
docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70),
range_facet_i_dv=DocValuesFormat(name=Asserting),
timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=237,
maxMBSortInHeap=7.164932465364936, sim=RandomSimilarity(queryNorm=true): {},
locale=mer, timezone=Africa/Conakry
[junit4] 2> NOTE: Linux 4.4.0-47-generic amd64/Oracle Corporation 9-ea
(64-bit)/cpus=12,threads=1,free=419193256,total=509280256
[junit4] 2> NOTE: All tests run in this JVM:
[TestNonDefinedSimilarityFactory, TestXmlQParserPlugin, TestReload,
TestFieldTypeResource, PreAnalyzedFieldTest, TestTolerantSearch,
TestExtendedDismaxParser, TestSlowCompositeReaderWrapper, FacetPivotSmallTest,
TestReqParamsAPI, SolrIndexConfigTest,
VMParamsZkACLAndCredentialsProvidersTest, TestManagedResource,
LukeRequestHandlerTest, DebugComponentTest, TestComplexPhraseQParserPlugin,
CoreAdminCreateDiscoverTest, HdfsLockFactoryTest, TestRTimerTree,
TemplateUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, TestSort,
CircularListTest, LeaderFailureAfterFreshStartTest]
[junit4] Completed [50/654 (1!)] on J2 in 59.60s, 1 test, 1 failure <<<
FAILURES!
[...truncated 53386 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
