Build: https://builds.apache.org/job/Lucene-Solr-tests-only-trunk/14942/

1 tests failed.
FAILED:  org.apache.solr.cloud.RecoveryZkTest.testDistribSearch

Error Message:
expected:<1677> but was:<1675>

Stack Trace:
java.lang.AssertionError: expected:<1677> but was:<1675>
        at 
__randomizedtesting.SeedInfo.seed([C933CEA48B6B772D:48D540BCFC341711]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at org.apache.solr.cloud.RecoveryZkTest.doTest(RecoveryZkTest.java:107)
        at 
org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:679)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:616)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1995)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.access$1100(RandomizedRunner.java:132)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:818)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:877)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
        at 
org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:32)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
        at 
org.apache.lucene.util.TestRuleReportUncaughtExceptions$1.evaluate(TestRuleReportUncaughtExceptions.java:68)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:825)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.access$700(RandomizedRunner.java:132)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$3$1.run(RandomizedRunner.java:671)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:697)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:736)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:747)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleReportUncaughtExceptions$1.evaluate(TestRuleReportUncaughtExceptions.java:68)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
org.apache.lucene.util.TestRuleIcuHack$1.evaluate(TestRuleIcuHack.java:51)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
        at 
org.apache.lucene.util.TestRuleNoInstanceHooksOverrides$1.evaluate(TestRuleNoInstanceHooksOverrides.java:53)
        at 
org.apache.lucene.util.TestRuleNoStaticHooksShadowing$1.evaluate(TestRuleNoStaticHooksShadowing.java:52)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:36)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:605)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.access$400(RandomizedRunner.java:132)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:551)




Build Log:
[...truncated 26793 lines...]
[junit4:junit4]   2> 30420 T988 C103 P56947 /update {wt=javabin&version=2} 
{delete=[1506 (-1408495194152632320)]} 0 2
[junit4:junit4]   2> 30422 T2835 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30422 T987 C103 P56947 /update {wt=javabin&version=2} 
{delete=[11505 (-1408495194154729472)]} 0 2
[junit4:junit4]   2> 30422 T972 C102 P49388 /update {wt=javabin&version=2} 
{add=[1617 (1408495194156826624)]} 0 0
[junit4:junit4]   2> 30424 T973 C102 P49388 /update {wt=javabin&version=2} 
{add=[11616 (1408495194158923776)]} 0 0
[junit4:junit4]   2> 30426 T2835 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30427 T987 C103 P56947 /update {wt=javabin&version=2} 
{add=[1617 (1408495194158923776)]} 0 3
[junit4:junit4]   2> 30428 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30428 T988 C103 P56947 /update {wt=javabin&version=2} 
{add=[11616 (1408495194161020928)]} 0 2
[junit4:junit4]   2> 30429 T973 C102 P49388 /update {wt=javabin&version=2} 
{add=[1618 (1408495194164166656)]} 0 0
[junit4:junit4]   2> 30430 T972 C102 P49388 /update {wt=javabin&version=2} 
{delete=[11506 (-1408495194165215232)]} 0 0
[junit4:junit4]   2> 30434 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30434 T987 C103 P56947 /update {wt=javabin&version=2} 
{delete=[11506 (-1408495194167312384)]} 0 2
[junit4:junit4]   2> 30436 T972 C102 P49388 /update {wt=javabin&version=2} 
{add=[11617 (1408495194171506688)]} 0 0
[junit4:junit4]   2> 30437 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30437 T988 C103 P56947 /update {wt=javabin&version=2} 
{add=[1618 (1408495194166263808)]} 0 6
[junit4:junit4]   2> 30439 T972 C102 P49388 /update {wt=javabin&version=2} 
{delete=[1508 (-1408495194174652416)]} 0 0
[junit4:junit4]   2> 30440 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30440 T987 C103 P56947 /update {wt=javabin&version=2} 
{add=[11617 (1408495194173603840)]} 0 2
[junit4:junit4]   2> 30442 T972 C102 P49388 /update {wt=javabin&version=2} 
{add=[11618 (1408495194177798144)]} 0 0
[junit4:junit4]   2> 30443 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30444 T988 C103 P56947 /update {wt=javabin&version=2} 
{delete=[1508 (-1408495194176749568)]} 0 3
[junit4:junit4]   2> 30446 T972 C102 P49388 /update {wt=javabin&version=2} 
{add=[1619 (1408495194181992448)]} 0 0
[junit4:junit4]   2> 30446 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30446 T987 C103 P56947 /update {wt=javabin&version=2} 
{add=[11618 (1408495194179895296)]} 0 2
[junit4:junit4]   2> 30449 T972 C102 P49388 /update {wt=javabin&version=2} 
{delete=[11508 (-1408495194185138176)]} 0 0
[junit4:junit4]   2> 30450 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30450 T988 C103 P56947 /update {wt=javabin&version=2} 
{add=[1619 (1408495194184089600)]} 0 2
[junit4:junit4]   2> 30452 T972 C102 P49388 /update {wt=javabin&version=2} 
{add=[1620 (1408495194188283904)]} 0 0
[junit4:junit4]   2> 30453 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30453 T987 C103 P56947 /update {wt=javabin&version=2} 
{delete=[11508 (-1408495194187235328)]} 0 2
[junit4:junit4]   2> 30453 T960 oascc.ZkStateReader.updateCloudState Updating 
cloud state from ZooKeeper... 
[junit4:junit4]   2> 30455 T972 C102 P49388 /update {wt=javabin&version=2} 
{add=[11619 (1408495194191429632)]} 0 0
[junit4:junit4]   2> 30456 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30456 T988 C103 P56947 /update {wt=javabin&version=2} 
{add=[1620 (1408495194190381056)]} 0 2
[junit4:junit4]   2> added docs:1620 with 1 fails deletes:781
[junit4:junit4]   2> 30459 T2834 C104 P29501 REQ /update 
{update.distrib=FROMLEADER&wt=javabin&version=2} status=0 QTime=0 
[junit4:junit4]   2> 30459 T987 C103 P56947 /update {wt=javabin&version=2} 
{add=[11619 (1408495194193526784)]} 0 2
[junit4:junit4]   2> added docs:1619 with 1 fails deletes:781
[junit4:junit4]   2> 30462 T972 C102 P49388 oasu.DirectUpdateHandler2.commit 
start 
commit{flags=0,_version_=0,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false}
[junit4:junit4]   2> 31015 T972 C102 P49388 oasc.SolrDeletionPolicy.onCommit 
SolrDeletionPolicy.onCommit: commits:num=2
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/control/data/index
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2773df16; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/control/data/index
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2773df16; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_2,generation=2,filenames=[_a4_Direct_0.frq, 
_a5.fdx, _a5.si, _a1.fnm, _a5.fdt, _a7_nrm.cfe, _a2_NestedPulsing_0.frq, 
_a2_Pulsing40_0.prx, _a8_nrm.cfs, _a7_Memory_0.ram, _a2.fdt, _a6_Direct_0.prx, 
_a2.fdx, _a1_Pulsing40_0.tim, _a7.fdt, _a5_Direct_0.prx, _a8_Pulsing40_0.frq, 
_a1_NestedPulsing_0.frq, _a6_NestedPulsing_0.tip, _a1_Memory_0.ram, 
_a2_NestedPulsing_0.tip, _a7_Pulsing40_0.tim, _a7.fdx, _a7_Pulsing40_0.tip, 
_a2_NestedPulsing_0.tim, _69_1.del, _a7_NestedPulsing_0.frq, 
_a6_Pulsing40_0.tip, _a6_NestedPulsing_0.prx, _a5_NestedPulsing_0.prx, 
_a6_Pulsing40_0.tim, _a1_Pulsing40_0.tip, _a5.fnm, _a8.fnm, _a2_Direct_0.tip, 
_a3.fdx, _a2_Direct_0.tim, _a3.fdt, _a7_NestedPulsing_0.tip, 
_a6_NestedPulsing_0.tim, _a3_Direct_0.prx, _a7_NestedPulsing_0.tim, _a7.si, 
_a3_Pulsing40_0.tip, _a4_Pulsing40_0.frq, _a8_NestedPulsing_0.tip, 
_a7_Pulsing40_0.prx, _a8.fdx, _a6_NestedPulsing_0.frq, _a4_NestedPulsing_0.frq, 
_a8_NestedPulsing_0.tim, _a3_Direct_0.frq, _a2_Memory_0.ram, 
_a3_Pulsing40_0.prx, _a1_Pulsing40_0.frq, _a4_Direct_0.prx, 
_a6_Pulsing40_0.frq, _69_Direct_0.prx, _a1.si, _a6_nrm.cfs, _a4_Memory_0.ram, 
_a7.fnm, _a2_Direct_0.frq, _69_Pulsing40_0.prx, _a1.fdt, _a2.fnm, 
_a5_NestedPulsing_0.frq, _a1_nrm.cfs, _a8_Direct_0.tip, _a1.fdx, 
_a3_Pulsing40_0.frq, _a5_Direct_0.tim, _a8_Pulsing40_0.prx, _a5_Direct_0.tip, 
_a5_Memory_0.ram, _a4_NestedPulsing_0.prx, _a8_NestedPulsing_0.frq, 
_a1_nrm.cfe, _a3.fnm, _a7_NestedPulsing_0.prx, _69_NestedPulsing_0.prx, 
_a8_Direct_0.tim, _a8.fdt, _a5_nrm.cfe, _a4_Pulsing40_0.prx, 
_69_Pulsing40_0.frq, _a5_nrm.cfs, _a2_1.del, _a5_Pulsing40_0.prx, 
_a3_Pulsing40_0.tim, _a6_Direct_0.tip, _a6.fnm, _a5_Pulsing40_0.frq, 
_a3_NestedPulsing_0.prx, _a6_Pulsing40_0.prx, _a4_NestedPulsing_0.tim, _a8.si, 
_a4_Pulsing40_0.tip, _a6_Memory_0.ram, _69.fdx, _a4_Pulsing40_0.tim, _a6.fdx, 
_a4_NestedPulsing_0.tip, _69.fdt, _69_NestedPulsing_0.frq, _a4.fnm, 
_a8_Memory_0.ram, _a6_Direct_0.tim, _a6_nrm.cfe, _69_Pulsing40_0.tim, 
_69_Pulsing40_0.tip, _a2_Pulsing40_0.frq, _a3_NestedPulsing_0.tip, _a6.fdt, 
_a3_NestedPulsing_0.tim, _a7_Direct_0.tim, _a7_Direct_0.tip, 
_a1_NestedPulsing_0.prx, _69_Direct_0.tip, _a6.si, _a5_Direct_0.frq, 
_69_NestedPulsing_0.tip, _69_Direct_0.tim, _a8_NestedPulsing_0.prx, 
_a1_NestedPulsing_0.tim, _a1_NestedPulsing_0.tip, _69_Memory_0.ram, 
_a3_Memory_0.ram, _a1_Direct_0.tip, _a8_Pulsing40_0.tip, _a8_Pulsing40_0.tim, 
_a1_Direct_0.tim, _a1_Pulsing40_0.prx, _a2_nrm.cfs, _a7_Direct_0.prx, 
_a5_NestedPulsing_0.tip, _69.si, _69_NestedPulsing_0.tim, 
_a2_NestedPulsing_0.prx, _a5_NestedPulsing_0.tim, _a2_Pulsing40_0.tip, _a4.fdx, 
_a2_nrm.cfe, _a2_Direct_0.prx, _a4.fdt, _a6_Direct_0.frq, _a3_Direct_0.tim, 
_a1_Direct_0.frq, _69_nrm.cfe, _a2_Pulsing40_0.tim, _a3_Direct_0.tip, 
_a8_Direct_0.prx, _a7_Direct_0.frq, _a1_Direct_0.prx, _69_nrm.cfs, 
_a3_NestedPulsing_0.frq, _a2.si, _a3_nrm.cfe, _a8_Direct_0.frq, _a4_nrm.cfs, 
_a4_Direct_0.tim, _a8_nrm.cfe, _69.fnm, _a3.si, _a4_Direct_0.tip, _a4.si, 
segments_2, _a7_nrm.cfs, _a5_Pulsing40_0.tip, _69_Direct_0.frq, _a3_nrm.cfs, 
_a7_Pulsing40_0.frq, _a4_nrm.cfe, _a5_Pulsing40_0.tim]
[junit4:junit4]   2> 31016 T972 C102 P49388 
oasc.SolrDeletionPolicy.updateCommits newest commit = 2
[junit4:junit4]   2> 31044 T972 C102 P49388 oass.SolrIndexSearcher.<init> 
Opening Searcher@2d8b1db4 main
[junit4:junit4]   2> 31045 T972 C102 P49388 oasu.DirectUpdateHandler2.commit 
end_commit_flush
[junit4:junit4]   2> 31045 T983 oasc.SolrCore.registerSearcher [collection1] 
Registered new searcher Searcher@2d8b1db4 
main{StandardDirectoryReader(segments_2:784 _69(5.0):C1026/64 _a2(5.0):C680/34 
_a1(5.0):C10 _a3(5.0):C10 _a4(5.0):C10 _a5(5.0):C10 _a6(5.0):C10 _a7(5.0):C10 
_a8(5.0):C9)}
[junit4:junit4]   2> 31057 T972 C102 P49388 /update 
{waitSearcher=true&wt=javabin&commit=true&softCommit=false&version=2} {commit=} 
0 595
[junit4:junit4]   2> 31059 T987 C103 P56947 oasu.DirectUpdateHandler2.commit 
start 
commit{flags=0,_version_=0,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false}
[junit4:junit4]   2> 31248 T987 C103 P56947 oasc.SolrDeletionPolicy.onCommit 
SolrDeletionPolicy.onCommit: commits:num=2
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/jetty1/index
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f2477f5; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_3,generation=3,filenames=[_96_Direct_0.frq, 
_95_NestedPulsing_0.frq, _96_Direct_0.tip, _96_Memory_0.ram, 
_96_NestedPulsing_0.frq, _95_Direct_0.tim, _96_NestedPulsing_0.prx, 
_95_Direct_0.tip, _95.si, _96_Direct_0.tim, _96_Direct_0.prx, _69.fdx, _69.fdt, 
_69_NestedPulsing_0.frq, _96.fnm, _95_Pulsing40_0.prx, _69_Pulsing40_0.tim, 
_96_Pulsing40_0.prx, _69_Pulsing40_0.tip, _69_1.del, _95_Direct_0.frq, 
_95_1.del, _96.fdx, _69_Direct_0.tip, _95_Pulsing40_0.tim, _69_Direct_0.tim, 
_69_NestedPulsing_0.tip, _95_Pulsing40_0.tip, _96.fdt, _69_Memory_0.ram, 
_96_nrm.cfs, _95_Pulsing40_0.frq, _96.si, _69.si, _95_Memory_0.ram, 
_95_NestedPulsing_0.prx, _69_NestedPulsing_0.tim, _69_Direct_0.prx, 
_95_nrm.cfs, _69_Pulsing40_0.prx, _95.fnm, _96_NestedPulsing_0.tip, 
_69_nrm.cfe, _95_Direct_0.prx, _95_nrm.cfe, _95_NestedPulsing_0.tip, 
_95_NestedPulsing_0.tim, _96_NestedPulsing_0.tim, _69_nrm.cfs, _96_nrm.cfe, 
_69_NestedPulsing_0.prx, _69.fnm, _96_Pulsing40_0.frq, _69_Pulsing40_0.frq, 
_96_Pulsing40_0.tip, _69_Direct_0.frq, _95.fdx, _96_Pulsing40_0.tim, 
segments_3, _95.fdt]
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/jetty1/index
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f2477f5; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_4,generation=4,filenames=[_a9_Direct_0.frq, 
_9d.fdt, _aa_NestedPulsing_0.tim, _9d.fdx, _aa_NestedPulsing_0.tip, 
_9d_NestedPulsing_0.frq, _a9.fnm, _aa_NestedPulsing_0.prx, _aa_Direct_0.prx, 
_a9_Direct_0.prx, _aa.fdx, _aa.fdt, _aa.si, _aa.fnm, _9d.fnm, _9d_1.del, 
_aa_Pulsing40_0.tim, _a9_NestedPulsing_0.frq, _aa_Pulsing40_0.tip, 
_a9_Direct_0.tim, _a9_Direct_0.tip, _a9_nrm.cfe, _a9_NestedPulsing_0.prx, 
_9d_nrm.cfe, _a9_Pulsing40_0.tip, _a9_NestedPulsing_0.tim, 
_a9_NestedPulsing_0.tip, _a9_nrm.cfs, _a9.si, _a9.fdx, _a9_Pulsing40_0.tim, 
_9d_NestedPulsing_0.tip, _9d_Direct_0.prx, _9d.si, _aa_Memory_0.ram, 
_a9_Pulsing40_0.prx, _9d_Pulsing40_0.frq, _9d_NestedPulsing_0.tim, _aa_nrm.cfe, 
_9d_nrm.cfs, _a9_1.del, _9d_Direct_0.frq, _aa_Direct_0.frq, 
_9d_Pulsing40_0.prx, _aa_nrm.cfs, _a9_Pulsing40_0.frq, _9d_Pulsing40_0.tim, 
_9d_Pulsing40_0.tip, _a9.fdt, _aa_NestedPulsing_0.frq, _9d_NestedPulsing_0.prx, 
_9d_Memory_0.ram, _aa_Direct_0.tip, _aa_Pulsing40_0.frq, _aa_Pulsing40_0.prx, 
_aa_Direct_0.tim, _9d_Direct_0.tim, _9d_Direct_0.tip, _a9_Memory_0.ram, 
segments_4]
[junit4:junit4]   2> 31249 T987 C103 P56947 
oasc.SolrDeletionPolicy.updateCommits newest commit = 4
[junit4:junit4]   2> 31269 T987 C103 P56947 oass.SolrIndexSearcher.<init> 
Opening Searcher@37b899d7 main
[junit4:junit4]   2> 31270 T987 C103 P56947 oasu.DirectUpdateHandler2.commit 
end_commit_flush
[junit4:junit4]   2> 31271 T995 oasc.SolrCore.registerSearcher [collection1] 
Registered new searcher Searcher@37b899d7 
main{StandardDirectoryReader(segments_4:789 _9d(5.0):C1522/104 _a9(5.0):C257/3 
_aa(5.0):C5)}
[junit4:junit4]   2> 31277 T2834 C104 P29501 oasu.DirectUpdateHandler2.commit 
start 
commit{flags=0,_version_=0,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false}
[junit4:junit4]   2> 31414 T2834 C104 P29501 oasc.SolrDeletionPolicy.onCommit 
SolrDeletionPolicy.onCommit: commits:num=2
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/jetty2/index.20120726014809276
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a50c03b; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_4,generation=4,filenames=[_9d.fdt, 
_9r_Direct_0.prx, _9d.fdx, _9r.fnm, _9o_Memory_0.ram, _9s_Pulsing40_0.prx, 
_9n_nrm.cfs, _9m_NestedPulsing_0.tim, _9p.fnm, _9t_NestedPulsing_0.prx, 
_9t_Direct_0.prx, _9o_nrm.cfs, _9r_NestedPulsing_0.tip, 
_9o_NestedPulsing_0.prx, _9p.fdx, _9v_nrm.cfe, _9t.fnm, _9s_Direct_0.frq, 
_9r_NestedPulsing_0.tim, _9p_Pulsing40_0.tim, _9s_NestedPulsing_0.prx, 
_9p_Pulsing40_0.tip, _9m_NestedPulsing_0.tip, _9s_NestedPulsing_0.frq, 
_9m_Direct_0.frq, _9q_Pulsing40_0.frq, _9d_1.del, _9v_nrm.cfs, 
_9p_NestedPulsing_0.prx, _9r_NestedPulsing_0.frq, _9r_Memory_0.ram, 
_9v_NestedPulsing_0.frq, _9o_Pulsing40_0.prx, _9p_Pulsing40_0.prx, 
_9t_Pulsing40_0.frq, _9n_nrm.cfe, _9r.fdt, _9n_Memory_0.ram, _9r.fdx, 
_9o_nrm.cfe, _9q_Memory_0.ram, _9d_nrm.cfe, _9q.fdx, _9m_nrm.cfs, _9t_nrm.cfs, 
_9p_nrm.cfs, _9n_NestedPulsing_0.tim, _9m_nrm.cfe, _9o.fdx, 
_9v_Pulsing40_0.frq, _9n_NestedPulsing_0.tip, _9d.si, _9d_NestedPulsing_0.tip, 
_9v_NestedPulsing_0.tim, _9r_nrm.cfs, _9q_nrm.cfe, _9n_Pulsing40_0.frq, 
_9s_nrm.cfs, _9v_NestedPulsing_0.tip, _9t.fdt, _9d_Pulsing40_0.frq, _9s.si, 
_9q.fnm, _9n_Direct_0.frq, _9d_NestedPulsing_0.tim, _9n.fdt, _9t.fdx, 
_9d_nrm.cfs, _9m_NestedPulsing_0.prx, _9s_Direct_0.tip, _9q_nrm.cfs, 
_9p_Direct_0.tim, _9s_nrm.cfe, _9t_NestedPulsing_0.tim, _9r_nrm.cfe, 
_9s_Direct_0.tim, _9d_Direct_0.frq, _9p_Direct_0.frq, _9t_NestedPulsing_0.tip, 
_9n_NestedPulsing_0.prx, _9v_Direct_0.frq, _9d_Pulsing40_0.prx, 
_9m_Pulsing40_0.prx, _9r_Direct_0.frq, _9n_Pulsing40_0.prx, _9o.fnm, 
_9o_Direct_0.prx, _9q_Direct_0.prx, _9s_NestedPulsing_0.tim, 
_9s_NestedPulsing_0.tip, _9m_NestedPulsing_0.frq, _9t_nrm.cfe, 
_9m_Pulsing40_0.tip, _9p_nrm.cfe, _9m_Direct_0.prx, _9m_Pulsing40_0.tim, 
_9p_Direct_0.tip, _9q_NestedPulsing_0.prx, _9q_Pulsing40_0.tip, 
_9t_Pulsing40_0.tim, _9q_Pulsing40_0.tim, _9o.fdt, _9o_Direct_0.tim, _9o.si, 
_9t_Pulsing40_0.tip, _9n_Direct_0.tim, _9d_NestedPulsing_0.frq, 
_9n_Pulsing40_0.tip, _9n.si, _9m_Memory_0.ram, _9o_Direct_0.tip, _9n.fnm, 
_9n_Pulsing40_0.tim, _9n_Direct_0.tip, _9v_Pulsing40_0.tim, _9s_Memory_0.ram, 
_9q_Direct_0.tip, _9t_Memory_0.ram, _9q_Direct_0.tim, _9v_Pulsing40_0.tip, 
_9d.fnm, _9t_Pulsing40_0.prx, _9q.fdt, _9p_NestedPulsing_0.frq, 
_9p_Pulsing40_0.frq, _9r_Direct_0.tim, _9r_Direct_0.tip, 
_9v_NestedPulsing_0.prx, _9n_Direct_0.prx, _9n.fdx, _9m.fdx, _9m.fdt, _9m.fnm, 
_9p_Memory_0.ram, _9v_Direct_0.tip, _9q.si, _9q_Direct_0.frq, 
_9o_NestedPulsing_0.frq, _9m.si, _9v_Direct_0.tim, _9p_Direct_0.prx, 
_9v_Pulsing40_0.prx, _9m_Pulsing40_0.frq, _9v.fdx, _9t.si, _9t_Direct_0.frq, 
_9o_Pulsing40_0.frq, _9v.si, _9r_NestedPulsing_0.prx, _9v.fdt, 
_9s_Pulsing40_0.frq, _9s.fnm, _9d_Direct_0.prx, _9o_NestedPulsing_0.tim, 
_9v_Memory_0.ram, _9o_NestedPulsing_0.tip, _9m_Direct_0.tim, 
_9q_NestedPulsing_0.frq, _9m_Direct_0.tip, _9s_Direct_0.prx, 
_9r_Pulsing40_0.frq, _9r_Pulsing40_0.prx, _9p_NestedPulsing_0.tim, 
_9v_Direct_0.prx, _9p_NestedPulsing_0.tip, _9n_NestedPulsing_0.frq, _9p.fdt, 
_9q_Pulsing40_0.prx, _9q_NestedPulsing_0.tim, _9r.si, _9v.fnm, _9s.fdx, 
_9t_NestedPulsing_0.frq, _9q_NestedPulsing_0.tip, _9s.fdt, _9r_Pulsing40_0.tip, 
_9d_Pulsing40_0.tim, _9d_Pulsing40_0.tip, _9d_Memory_0.ram, 
_9d_NestedPulsing_0.prx, _9o_Pulsing40_0.tim, _9p.si, _9o_Pulsing40_0.tip, 
_9s_Pulsing40_0.tip, _9d_Direct_0.tim, _9t_Direct_0.tip, _9s_Pulsing40_0.tim, 
_9o_Direct_0.frq, _9d_Direct_0.tip, _9r_Pulsing40_0.tim, _9t_Direct_0.tim, 
segments_4]
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/jetty2/index.20120726014809276
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a50c03b; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_5,generation=5,filenames=[_9d.fdt, 
_aa_NestedPulsing_0.tim, _9d.fdx, _aa_NestedPulsing_0.tip, 
_9d_NestedPulsing_0.frq, _ab_Pulsing40_0.frq, _aa_NestedPulsing_0.prx, 
_ab_Direct_0.tip, _aa_Direct_0.prx, _ab_Pulsing40_0.prx, _aa.fdx, 
_ab_Direct_0.tim, _aa.fdt, _aa.si, _aa.fnm, _9d.fnm, _ab_NestedPulsing_0.frq, 
_aa_Pulsing40_0.tim, _aa_Pulsing40_0.tip, _ab_NestedPulsing_0.prx, _9d_2.del, 
_ab_Direct_0.prx, _9d_nrm.cfe, _ab_NestedPulsing_0.tip, 
_ab_NestedPulsing_0.tim, _ab_Direct_0.frq, _ab.si, _9d_NestedPulsing_0.tip, 
_9d_Direct_0.prx, _9d.si, _aa_Memory_0.ram, _ab_nrm.cfe, _ab_1.del, 
_9d_Pulsing40_0.frq, _9d_NestedPulsing_0.tim, _aa_nrm.cfe, _9d_nrm.cfs, 
_ab_nrm.cfs, _ab_Memory_0.ram, _9d_Direct_0.frq, _aa_Direct_0.frq, 
_9d_Pulsing40_0.prx, _aa_nrm.cfs, _ab_Pulsing40_0.tip, _9d_Pulsing40_0.tim, 
_9d_Pulsing40_0.tip, _ab_Pulsing40_0.tim, _aa_NestedPulsing_0.frq, 
_9d_NestedPulsing_0.prx, _9d_Memory_0.ram, _ab.fdt, _aa_Direct_0.tip, 
_aa_Pulsing40_0.frq, _aa_Pulsing40_0.prx, _aa_Direct_0.tim, _ab.fdx, 
_9d_Direct_0.tim, _ab.fnm, segments_5, _9d_Direct_0.tip]
[junit4:junit4]   2> 31416 T2834 C104 P29501 
oasc.SolrDeletionPolicy.updateCommits newest commit = 5
[junit4:junit4]   2> 31442 T2834 C104 P29501 oass.SolrIndexSearcher.<init> 
Opening Searcher@118d8916 main
[junit4:junit4]   2> 31443 T2834 C104 P29501 oasu.DirectUpdateHandler2.commit 
end_commit_flush
[junit4:junit4]   2> 31443 T2917 oasc.SolrCore.registerSearcher [collection1] 
Registered new searcher Searcher@118d8916 
main{StandardDirectoryReader(segments_5:790:nrt _9d(5.0):C1522/104 
_ab(5.0):C253/5 _aa(5.0):C9)}
[junit4:junit4]   2> 31444 T2834 C104 P29501 /update 
{waitSearcher=true&commit=true&wt=javabin&commit_end_point=true&softCommit=false&version=2}
 {commit=} 0 167
[junit4:junit4]   2> 31444 T987 C103 P56947 /update 
{waitSearcher=true&wt=javabin&commit=true&softCommit=false&version=2} {commit=} 
0 385
[junit4:junit4]   2> 31446 T990 C103 P56947 REQ /select 
{distrib=false&wt=javabin&q=*:*&version=2} hits=1677 status=0 QTime=1 
[junit4:junit4]   2> 31448 T2890 C104 P29501 REQ /select 
{distrib=false&wt=javabin&q=*:*&version=2} hits=1675 status=0 QTime=1 
[junit4:junit4]   2> 31452 T990 C103 P56947 REQ /select 
{distrib=false&wt=javabin&q=*:*&version=2} hits=1677 status=0 QTime=0 
[junit4:junit4]   2> 31454 T2890 C104 P29501 REQ /select 
{distrib=false&wt=javabin&q=*:*&version=2} hits=1675 status=0 QTime=0 
[junit4:junit4]   2> 31455 T965 oazs.PrepRequestProcessor.pRequest Processed 
session termination for sessionid: 0x138bfae336b0006
[junit4:junit4]   2> 31456 T1014 oaz.ClientCnxn$EventThread.run EventThread 
shut down
[junit4:junit4]   2> 31456 T960 oaz.ZooKeeper.close Session: 0x138bfae336b0006 
closed
[junit4:junit4]   2> 31457 T962 oazs.NIOServerCnxn.closeSock Closed socket 
connection for client /140.211.11.196:14232 which had sessionid 
0x138bfae336b0006
[junit4:junit4]   2> 31458 T965 oazs.PrepRequestProcessor.pRequest Processed 
session termination for sessionid: 0x138bfae336b0005
[junit4:junit4]   2> 31459 T1010 oaz.ClientCnxn$EventThread.run EventThread 
shut down
[junit4:junit4]   2> 31459 T960 oaz.ZooKeeper.close Session: 0x138bfae336b0005 
closed
[junit4:junit4]   2> 31459 T962 oazs.NIOServerCnxn.closeSock Closed socket 
connection for client /140.211.11.196:13411 which had sessionid 
0x138bfae336b0005
[junit4:junit4]   2> 31459 T965 oazs.PrepRequestProcessor.run 
PrepRequestProcessor exited loop!
[junit4:junit4]   2> 31459 T964 oazs.SyncRequestProcessor.run 
SyncRequestProcessor exited!
[junit4:junit4]   2> 31460 T960 oazs.FinalRequestProcessor.shutdown shutdown of 
request processor complete
[junit4:junit4]   2> 31591 T960 oazs.NIOServerCnxn.closeSock Closed socket 
connection for client /140.211.11.196:61232 which had sessionid 
0x138bfae336b0003
[junit4:junit4]   2> 31591 T993 oaz.ClientCnxn$SendThread.run Unable to read 
additional data from server sessionid 0x138bfae336b0003, likely server has 
closed socket, closing socket connection and attempting reconnect
[junit4:junit4]   2> 31591 T960 oazs.NIOServerCnxn.closeSock Closed socket 
connection for client /140.211.11.196:10762 which had sessionid 
0x138bfae336b0002
[junit4:junit4]   2> 31591 T979 oaz.ClientCnxn$SendThread.run Unable to read 
additional data from server sessionid 0x138bfae336b0002, likely server has 
closed socket, closing socket connection and attempting reconnect
[junit4:junit4]   2> 31591 T960 oazs.NIOServerCnxn.closeSock Closed socket 
connection for client /140.211.11.196:31979 which had sessionid 
0x138bfae336b0007
[junit4:junit4]   2> 31592 T2839 oaz.ClientCnxn$SendThread.run Unable to read 
additional data from server sessionid 0x138bfae336b0007, likely server has 
closed socket, closing socket connection and attempting reconnect
[junit4:junit4]   2> 31592 T962 oazs.NIOServerCnxn$Factory.run NIOServerCnxn 
factory exited run method
[junit4:junit4]   2> 31593 T960 oazs.FinalRequestProcessor.shutdown shutdown of 
request processor complete
[junit4:junit4]   2> 31593 T960 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 
49388
[junit4:junit4]   2> 31593 T960 oasc.CoreContainer.shutdown Shutting down 
CoreContainer instance=1515657526
[junit4:junit4]   2> 31593 T960 oasc.SolrCore.close [collection1]  CLOSING 
SolrCore org.apache.solr.core.SolrCore@5d2669e0
[junit4:junit4]   2> 31594 T960 oasc.SolrCore.closeSearcher [collection1] 
Closing main searcher on request.
[junit4:junit4]   2> 31596 T960 oasu.DirectUpdateHandler2.close closing 
DirectUpdateHandler2{commits=1,autocommits=0,soft 
autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3239,cumulative_deletesById=1562,cumulative_deletesByQuery=0,cumulative_errors=0}
[junit4:junit4]   2> 31673 T960 C102 P49388 oasc.SolrDeletionPolicy.onCommit 
SolrDeletionPolicy.onCommit: commits:num=2
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/control/data/index
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2773df16; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_2,generation=2,filenames=[_a4_Direct_0.frq, 
_a5.fdx, _a5.si, _a1.fnm, _a5.fdt, _a7_nrm.cfe, _a2_NestedPulsing_0.frq, 
_a2_Pulsing40_0.prx, _a8_nrm.cfs, _a7_Memory_0.ram, _a2.fdt, _a6_Direct_0.prx, 
_a2.fdx, _a1_Pulsing40_0.tim, _a7.fdt, _a5_Direct_0.prx, _a8_Pulsing40_0.frq, 
_a1_NestedPulsing_0.frq, _a6_NestedPulsing_0.tip, _a1_Memory_0.ram, 
_a2_NestedPulsing_0.tip, _a7_Pulsing40_0.tim, _a7.fdx, _a7_Pulsing40_0.tip, 
_a2_NestedPulsing_0.tim, _69_1.del, _a7_NestedPulsing_0.frq, 
_a6_Pulsing40_0.tip, _a6_NestedPulsing_0.prx, _a5_NestedPulsing_0.prx, 
_a6_Pulsing40_0.tim, _a1_Pulsing40_0.tip, _a5.fnm, _a8.fnm, _a2_Direct_0.tip, 
_a3.fdx, _a2_Direct_0.tim, _a3.fdt, _a7_NestedPulsing_0.tip, 
_a6_NestedPulsing_0.tim, _a3_Direct_0.prx, _a7_NestedPulsing_0.tim, _a7.si, 
_a3_Pulsing40_0.tip, _a4_Pulsing40_0.frq, _a8_NestedPulsing_0.tip, 
_a7_Pulsing40_0.prx, _a8.fdx, _a6_NestedPulsing_0.frq, _a4_NestedPulsing_0.frq, 
_a8_NestedPulsing_0.tim, _a3_Direct_0.frq, _a2_Memory_0.ram, 
_a3_Pulsing40_0.prx, _a1_Pulsing40_0.frq, _a4_Direct_0.prx, 
_a6_Pulsing40_0.frq, _69_Direct_0.prx, _a1.si, _a6_nrm.cfs, _a4_Memory_0.ram, 
_a7.fnm, _a2_Direct_0.frq, _69_Pulsing40_0.prx, _a1.fdt, _a2.fnm, 
_a5_NestedPulsing_0.frq, _a1_nrm.cfs, _a8_Direct_0.tip, _a1.fdx, 
_a3_Pulsing40_0.frq, _a5_Direct_0.tim, _a8_Pulsing40_0.prx, _a5_Direct_0.tip, 
_a5_Memory_0.ram, _a4_NestedPulsing_0.prx, _a8_NestedPulsing_0.frq, 
_a1_nrm.cfe, _a3.fnm, _a7_NestedPulsing_0.prx, _69_NestedPulsing_0.prx, 
_a8_Direct_0.tim, _a8.fdt, _a5_nrm.cfe, _a4_Pulsing40_0.prx, 
_69_Pulsing40_0.frq, _a5_nrm.cfs, _a2_1.del, _a5_Pulsing40_0.prx, 
_a3_Pulsing40_0.tim, _a6_Direct_0.tip, _a6.fnm, _a5_Pulsing40_0.frq, 
_a3_NestedPulsing_0.prx, _a6_Pulsing40_0.prx, _a4_NestedPulsing_0.tim, _a8.si, 
_a4_Pulsing40_0.tip, _a6_Memory_0.ram, _69.fdx, _a4_Pulsing40_0.tim, _a6.fdx, 
_a4_NestedPulsing_0.tip, _69.fdt, _69_NestedPulsing_0.frq, _a4.fnm, 
_a8_Memory_0.ram, _a6_Direct_0.tim, _a6_nrm.cfe, _69_Pulsing40_0.tim, 
_69_Pulsing40_0.tip, _a2_Pulsing40_0.frq, _a3_NestedPulsing_0.tip, _a6.fdt, 
_a3_NestedPulsing_0.tim, _a7_Direct_0.tim, _a7_Direct_0.tip, 
_a1_NestedPulsing_0.prx, _69_Direct_0.tip, _a6.si, _a5_Direct_0.frq, 
_69_NestedPulsing_0.tip, _69_Direct_0.tim, _a8_NestedPulsing_0.prx, 
_a1_NestedPulsing_0.tim, _a1_NestedPulsing_0.tip, _69_Memory_0.ram, 
_a3_Memory_0.ram, _a1_Direct_0.tip, _a8_Pulsing40_0.tip, _a8_Pulsing40_0.tim, 
_a1_Direct_0.tim, _a1_Pulsing40_0.prx, _a2_nrm.cfs, _a7_Direct_0.prx, 
_a5_NestedPulsing_0.tip, _69.si, _69_NestedPulsing_0.tim, 
_a2_NestedPulsing_0.prx, _a5_NestedPulsing_0.tim, _a2_Pulsing40_0.tip, _a4.fdx, 
_a2_nrm.cfe, _a2_Direct_0.prx, _a4.fdt, _a6_Direct_0.frq, _a3_Direct_0.tim, 
_a1_Direct_0.frq, _69_nrm.cfe, _a2_Pulsing40_0.tim, _a3_Direct_0.tip, 
_a8_Direct_0.prx, _a7_Direct_0.frq, _a1_Direct_0.prx, _69_nrm.cfs, 
_a3_NestedPulsing_0.frq, _a2.si, _a3_nrm.cfe, _a8_Direct_0.frq, _a4_nrm.cfs, 
_a4_Direct_0.tim, _a8_nrm.cfe, _69.fnm, _a3.si, _a4_Direct_0.tip, _a4.si, 
segments_2, _a7_nrm.cfs, _a5_Pulsing40_0.tip, _69_Direct_0.frq, _a3_nrm.cfs, 
_a7_Pulsing40_0.frq, _a4_nrm.cfe, _a5_Pulsing40_0.tim]
[junit4:junit4]   2>            
commit{dir=NRTCachingDirectory(org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build/solr-core/test/J0/org.apache.solr.cloud.RecoveryZkTest-1343245661029/control/data/index
 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2773df16; 
maxCacheMB=48.0 
maxMergeSizeMB=4.0),segFN=segments_3,generation=3,filenames=[_a9_NestedPulsing_0.prx,
 _a9_Direct_0.frq, _a9_Pulsing40_0.tip, _a9_NestedPulsing_0.tim, 
_a9_NestedPulsing_0.tip, _a8_Pulsing40_0.tip, _a8_Pulsing40_0.tim, _a9_nrm.cfs, 
_a8_NestedPulsing_0.tip, _a8.fdx, _a8_nrm.cfs, _a9.si, _a8_NestedPulsing_0.tim, 
_a9.fdx, _a9_Pulsing40_0.tim, _a9.fnm, _a8.si, _a9_Pulsing40_0.prx, 
_a9_Direct_0.prx, _a8_Memory_0.ram, _a8_Pulsing40_0.frq, _a8_Direct_0.prx, 
_a8_Direct_0.tip, _a9_Pulsing40_0.frq, _a8_Pulsing40_0.prx, 
_a8_NestedPulsing_0.frq, _a8_Direct_0.frq, _a9_NestedPulsing_0.frq, _a8.fnm, 
_a8_nrm.cfe, _a9.fdt, _a8_Direct_0.tim, _a8.fdt, _a9_Direct_0.tim, 
_a9_Direct_0.tip, _a8_NestedPulsing_0.prx, segments_3, _a9_Memory_0.ram, 
_a9_nrm.cfe]
[junit4:junit4]   2> 31674 T960 C102 P49388 
oasc.SolrDeletionPolicy.updateCommits newest commit = 3
[junit4:junit4]   2> 31692 T994 oascc.ConnectionManager.process Watcher 
org.apache.solr.common.cloud.ConnectionManager@1d5dcc8c 
name:ZooKeeperConnection Watcher:127.0.0.1:65108/solr got event WatchedEvent 
state:Disconnected type:None path:null path:null type:None
[junit4:junit4]   2> 31693 T2840 oascc.ConnectionManager.process Watcher 
org.apache.solr.common.cloud.ConnectionManager@71490a6d 
name:ZooKeeperConnection Watcher:127.0.0.1:65108/solr got event WatchedEvent 
state:Disconnected type:None path:null path:null type:None
[junit4:junit4]   2> 31693 T960 oaz.ZooKeeper.close Session: 0x138bfae336b0002 
closed
[junit4:junit4]   2> 31693 T980 oascc.ConnectionManager.process Watcher 
org.apache.solr.common.cloud.ConnectionManager@c385690 name:ZooKeeperConnection 
Watcher:127.0.0.1:65108/solr got event WatchedEvent state:Disconnected 
type:None path:null path:null type:None
[junit4:junit4]   2> 31693 T980 oaz.ClientCnxn$EventThread.run EventThread shut 
down
[junit4:junit4]   2> 31694 T982 oasc.OverseerCollectionProcessor.run WARNING 
Overseer cannot talk to ZK
[junit4:junit4]   2> 31694 T960 oejsh.ContextHandler.doStop stopped 
o.e.j.s.ServletContextHandler{/solr,null}
[junit4:junit4]   2> 31746 T960 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 
56947
[junit4:junit4]   2> 31746 T960 oasc.CoreContainer.shutdown Shutting down 
CoreContainer instance=285985666
[junit4:junit4]   2> 31746 T960 oasc.SolrCore.close [collection1]  CLOSING 
SolrCore org.apache.solr.core.SolrCore@8669f00
[junit4:junit4]   2> 31747 T960 oasc.SolrCore.closeSearcher [collection1] 
Closing main searcher on request.
[junit4:junit4]   2> 31748 T960 oasu.DirectUpdateHandler2.close closing 
DirectUpdateHandler2{commits=3,autocommits=0,soft 
autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3239,cumulative_deletesById=1562,cumulative_deletesByQuery=0,cumulative_errors=0}
[junit4:junit4]   2> 31913 T981 oasc.Overseer$CloudStateUpdater.amILeader 
WARNING  org.apache.zookeeper.KeeperException$SessionExpiredException: 
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4:junit4]   2>    at 
org.apache.zookeeper.KeeperException.create(KeeperException.java:118)
[junit4:junit4]   2>    at 
org.apache.zookeeper.KeeperException.create(KeeperException.java:42)
[junit4:junit4]   2>    at 
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:927)
[junit4:junit4]   2>    at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:289)
[junit4:junit4]   2>    at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:286)
[junit4:junit4]   2>    at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:67)
[junit4:junit4]   2>    at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:286)
[junit4:junit4]   2>    at 
org.apache.solr.cloud.Overseer$CloudStateUpdater.amILeader(Overseer.java:187)
[junit4:junit4]   2>    at 
org.apache.solr.cloud.Overseer$CloudStateUpdater.run(Overseer.java:113)
[junit4:junit4]   2>    at java.lang.Thread.run(Thread.java:679)
[junit4:junit4]   2> 
[junit4:junit4]   2> 31913 T981 oasc.Overseer$CloudStateUpdater.amILeader 
According to ZK I (id=88030947641589762-127.0.0.1:49388_solr-n_0000000000) am 
no longer a leader.
[junit4:junit4]   2> 32843 T993 oaz.ClientCnxn$SendThread.startConnect Opening 
socket connection to server localhost/127.0.0.1:65108
[junit4:junit4]   2> 33008 T2839 oaz.ClientCnxn$SendThread.startConnect Opening 
socket connection to server localhost/127.0.0.1:65108
[junit4:junit4]   2> 33974 T963 oazs.SessionTrackerImpl.run SessionTrackerImpl 
exited loop!
[junit4:junit4]   2> 43694 T2839 oaz.ClientCnxn$SendThread.run Client session 
timed out, have not heard from server in 12001ms for sessionid 
0x138bfae336b0007, closing socket connection and attempting reconnect
[junit4:junit4]   2> 43794 T994 oaz.ClientCnxn$EventThread.run EventThread shut 
down
[junit4:junit4]   2> 43794 T960 oaz.ZooKeeper.close Session: 0x138bfae336b0003 
closed
[junit4:junit4]   2> 43795 T960 oejsh.ContextHandler.doStop stopped 
o.e.j.s.ServletContextHandler{/solr,null}
[junit4:junit4]   2> 43847 T960 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 
29501
[junit4:junit4]   2> 43847 T960 oasc.CoreContainer.shutdown Shutting down 
CoreContainer instance=663962491
[junit4:junit4]   2> 43847 T960 oasc.RecoveryStrategy.close WARNING Stopping 
recovery for core collection1 zkNodeName=127.0.0.1:29501_solr_collection1
[junit4:junit4]   2> 43847 T960 oasc.SolrCore.close [collection1]  CLOSING 
SolrCore org.apache.solr.core.SolrCore@72452e82
[junit4:junit4]   2> 43849 T960 oasc.SolrCore.closeSearcher [collection1] 
Closing main searcher on request.
[junit4:junit4]   2> 43849 T960 oasu.DirectUpdateHandler2.close closing 
DirectUpdateHandler2{commits=3,autocommits=0,soft 
autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1592,cumulative_deletesById=772,cumulative_deletesByQuery=0,cumulative_errors=0}
[junit4:junit4]   2> 43850 T960 oasc.RecoveryStrategy.close WARNING Stopping 
recovery for core collection1 zkNodeName=127.0.0.1:29501_solr_collection1
[junit4:junit4]   2> 45265 T2839 oaz.ClientCnxn$SendThread.startConnect Opening 
socket connection to server localhost/127.0.0.1:65108
[junit4:junit4]   2> 55897 T2840 oaz.ClientCnxn$EventThread.run EventThread 
shut down
[junit4:junit4]   2> 55897 T960 oaz.ZooKeeper.close Session: 0x138bfae336b0007 
closed
[junit4:junit4]   2> 55898 T960 oejsh.ContextHandler.doStop stopped 
o.e.j.s.ServletContextHandler{/solr,null}
[junit4:junit4]   2> 55980 T960 oas.SolrTestCaseJ4.tearDown ###Ending 
testDistribSearch
[junit4:junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest 
-Dtests.method=testDistribSearch -Dtests.seed=C933CEA48B6B772D 
-Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_CR 
-Dtests.timezone=Asia/Thimphu -Dtests.file.encoding=US-ASCII
[junit4:junit4]   2>
[junit4:junit4]    > (@AfterClass output)
[junit4:junit4]   2> 56007 T960 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4]   2> NOTE: test params are: codec=Lucene40: 
{range_facet_l=PostingsFormat(name=Direct), 
text=PostingsFormat(name=NestedPulsing), _version_=PostingsFormat(name=Memory 
doPackFST= false), rnd_b=Pulsing40(freqCutoff=15 minBlockSize=32 
maxBlockSize=92), intDefault=PostingsFormat(name=Memory doPackFST= false), 
id=Pulsing40(freqCutoff=15 minBlockSize=32 maxBlockSize=92), 
timestamp=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), 
range_facet_sl=Pulsing40(freqCutoff=15 minBlockSize=32 maxBlockSize=92), 
range_facet_si=PostingsFormat(name=Direct), 
other_tl1=PostingsFormat(name=Memory doPackFST= false), 
multiDefault=PostingsFormat(name=Direct), a_si=PostingsFormat(name=Memory 
doPackFST= false)}, sim=DefaultSimilarity, locale=es_CR, timezone=Asia/Thimphu
[junit4:junit4]   2> NOTE: FreeBSD 9.0-RELEASE amd64/Sun Microsystems Inc. 
1.6.0_32 (64-bit)/cpus=16,threads=4,free=117456776,total=326500352
[junit4:junit4]   2> NOTE: All tests run in this JVM: [CacheHeaderTest, 
DistanceFunctionTest, TestStressReorder, SOLR749Test, 
SpellingQueryConverterTest, BinaryUpdateRequestHandlerTest, 
AnalysisAfterCoreReloadTest, PingRequestHandlerTest, 
DirectSolrSpellCheckerTest, TestDefaultSimilarityFactory, 
XmlUpdateRequestHandlerTest, TermVectorComponentTest, 
DefaultValueUpdateProcessorTest, DirectSolrConnectionTest, 
TestMultiCoreConfBootstrap, TestSearchPerf, TestRandomFaceting, FileUtilsTest, 
LeaderElectionIntegrationTest, DistributedTermsComponentTest, 
TestDistributedGrouping, PeerSyncTest, SpellPossibilityIteratorTest, 
SolrCmdDistributorTest, TestDistributedSearch, SolrCoreCheckLockOnStartupTest, 
TestLMDirichletSimilarityFactory, DOMUtilTest, SystemInfoHandlerTest, 
SortByFunctionTest, MBeansHandlerTest, PluginInfoTest, 
IndexSchemaRuntimeFieldTest, ZkSolrClientTest, TestReload, 
TestIBSimilarityFactory, TestExtendedDismaxParser, 
UniqFieldsUpdateProcessorFactoryTest, HighlighterConfigTest, 
AlternateDirectoryTest, MinimalSchemaTest, TestValueSourceCache, 
StandardRequestHandlerTest, NotRequiredUniqueKeyTest, RecoveryZkTest]
[junit4:junit4]   2> 
[junit4:junit4] Completed on J0 in 56.02s, 1 test, 1 failure <<< FAILURES!
[...truncated 471 lines...]

[...truncated 27415 lines...]

[...truncated 27415 lines...]

[...truncated 27415 lines...]

[...truncated 27415 lines...]

[...truncated 27400 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/build.xml:29:
 The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/solr/build.xml:144:
 The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/lucene/module-build.xml:62:
 The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/lucene/common-build.xml:1094:
 The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-tests-only-trunk/checkout/lucene/common-build.xml:789:
 There were test failures: 222 suites, 896 tests, 1 failure, 5 ignored

Total time: 25 minutes 21 seconds
Build step 'Execute shell' marked build as failure
Archiving artifacts
Recording test results
Publishing Clover coverage report...
No Clover report will be published due to a Build Failure
Email was triggered for: Failure
Sending email for trigger: Failure


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to