[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15052643#comment-15052643
]
Michael McCandless commented on LUCENE-6629:
Again? http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/75411/
{noformat}
[junit4] Suite: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3
[junit4] 2> 12? 11, 2015 2:18:40 ??
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out:
org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3
[junit4] 2>1) Thread[id=11,
name=TEST-TestBlockPostingsFormat3.test-seed#[45106F11E17A4109],
state=RUNNABLE, group=TGRP-TestBlockPostingsFormat3]
[junit4] 2> at
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.next(SegmentTermsEnum.java:894)
[junit4] 2> at
org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.assertTermsSeeking(TestBlockPostingsFormat3.java:211)
[junit4] 2> at
org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.assertTerms(TestBlockPostingsFormat3.java:180)
[junit4] 2> at
org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.verify(TestBlockPostingsFormat3.java:153)
[junit4] 2> at
org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.test(TestBlockPostingsFormat3.java:145)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2> at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
[junit4] 2> at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:606)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
[junit4] 2> at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2> at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2> at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2> at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
[junit4] 2> at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15037691#comment-15037691
]
Michael McCandless commented on LUCENE-6629:
Maybe another one?
http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/74226/
{noformat}
[junit4] Suite:
org.apache.lucene.codecs.lucene41.TestLucene41StoredFieldsFormat
[junit4] 2> gru 03, 2015 12:32:30 AM
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out:
org.apache.lucene.codecs.lucene41.TestLucene41StoredFieldsFormat
[junit4] 2>1) Thread[id=70,
name=SUITE-TestLucene41StoredFieldsFormat-seed#[1B32743AE8AA0DDD],
state=RUNNABLE, group=TGRP-TestLucene41StoredFieldsFormat]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1552)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
[junit4] 2> at java.security.AccessController.doPrivileged(Native
Method)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
[junit4] 2>2) Thread[id=10, name=JUnit4-serializer-daemon,
state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
[junit4] 2>3) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1245)
[junit4] 2> at java.lang.Thread.join(Thread.java:1319)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:244)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
[junit4] 2>4) Thread[id=71,
name=TEST-TestLucene41StoredFieldsFormat.testMergeLargeDocuments-seed#[1B32743AE8AA0DDD],
state=RUNNABLE, group=TGRP-TestLucene41StoredFieldsFormat]
[junit4] 2> at
org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
[junit4] 2> at
org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsWriter.flush(Lucene41StoredFieldsWriter.java:218)
[junit4] 2> at
org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsWriter.finishDocument(Lucene41StoredFieldsWriter.java:151)
[junit4] 2> at
org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:270)
[junit4] 2> at
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:311)
[junit4] 2> at
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234)
[junit4] 2> at
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450)
[junit4] 2> at
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477)
[junit4] 2> at
org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256)
[junit4] 2> at
org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:170)
[junit4] 2> at
org.apache.lucene.codecs.lucene41.TestLucene41StoredFieldsFormat.testMergeLargeDocuments(TestLucene41StoredFieldsFormat.java:70)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2> at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=14734503#comment-14734503
]
Michael McCandless commented on LUCENE-6629:
Another one: http://jenkins.thetaphi.de/job/Lucene-Solr-5.x-Linux/13870/
{noformat}
Suite: org.apache.lucene.analysis.util.TestCharArrayMap
[junit4] 2> сеп 07, 2015 11:56:46 AM
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out:
org.apache.lucene.analysis.util.TestCharArrayMap
[junit4] 2>1) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1245)
[junit4] 2> at java.lang.Thread.join(Thread.java:1319)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 2>2) Thread[id=11, name=JUnit4-serializer-daemon,
state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
[junit4] 2>3) Thread[id=840,
name=TEST-TestCharArrayMap.testCharArrayMap-seed#[9E827E74BAD348B7],
state=RUNNABLE, group=TGRP-TestCharArrayMap]
[junit4] 2> at
org.apache.lucene.analysis.util.CharArrayMap.getSlot(CharArrayMap.java:166)
[junit4] 2> at
org.apache.lucene.analysis.util.CharArrayMap.get(CharArrayMap.java:128)
[junit4] 2> at
org.apache.lucene.analysis.util.TestCharArrayMap.doRandom(TestCharArrayMap.java:52)
[junit4] 2> at
org.apache.lucene.analysis.util.TestCharArrayMap.testCharArrayMap(TestCharArrayMap.java:62)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2> at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
[junit4] 2> at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2> at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2> at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2> at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
[junit4] 2> at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=14734549#comment-14734549
]
Michael McCandless commented on LUCENE-6629:
Another one:
http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/62917
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRangeQuery
[junit4] 2> 9 08, 2015 10:21:29 ??
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRangeQuery
[junit4] 2>1) Thread[id=11, name=JUnit4-serializer-daemon,
state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
[junit4] 2>2) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1245)
[junit4] 2> at java.lang.Thread.join(Thread.java:1319)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2> at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 2>3) Thread[id=14,
name=SUITE-TestDocValuesRangeQuery-seed#[57620327D30425D5], state=RUNNABLE,
group=TGRP-TestDocValuesRangeQuery]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1552)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62)
[junit4] 2> at
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568)
[junit4] 2>4) Thread[id=15,
name=TEST-TestDocValuesRangeQuery.testScore-seed#[57620327D30425D5],
state=RUNNABLE, group=TGRP-TestDocValuesRangeQuery]
[junit4] 2> at
org.apache.lucene.search.TopFieldCollector.populateResults(TopFieldCollector.java:537)
[junit4] 2> at
org.apache.lucene.search.TopDocsCollector.topDocs(TopDocsCollector.java:156)
[junit4] 2> at
org.apache.lucene.search.TopDocsCollector.topDocs(TopDocsCollector.java:93)
[junit4] 2> at
org.apache.lucene.search.TopFieldCollector.topDocs(TopFieldCollector.java:561)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher$4.reduce(IndexSearcher.java:696)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher$4.reduce(IndexSearcher.java:683)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:719)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher.searchAfter(IndexSearcher.java:703)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher.searchAfter(IndexSearcher.java:645)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:551)
[junit4] 2> at
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:601)
[junit4] 2> at
org.apache.lucene.search.TestDocValuesRangeQuery.assertSameMatches(TestDocValuesRangeQuery.java:230)
[junit4] 2> at
org.apache.lucene.search.TestDocValuesRangeQuery.testScore(TestDocValuesRangeQuery.java:169)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2> at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[junit4] 2> at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
[junit4] 2>
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14663006#comment-14663006
]
Michael McCandless commented on LUCENE-6629:
Another strange timeout:
http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/58384/console
{noformat}
[junit4] Suite: org.apache.lucene.search.suggest.TestInputIterator
[junit4] 2 авг 08, 2015 3:57:45 AM
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.suggest.TestInputIterator
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=12,
name=SUITE-TestInputIterator-seed#[4568F9BCA17391B], state=RUNNABLE,
group=TGRP-TestInputIterator]
[junit4] 2 at java.lang.Thread.getStackTrace(Thread.java:1552)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568)
[junit4] 23) Thread[id=11, name=JUnit4-serializer-daemon,
state=TIMED_WAITING, group=main]
[junit4] 2 at java.lang.Thread.sleep(Native Method)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
[junit4] 24) Thread[id=13,
name=TEST-TestInputIterator.testTerms-seed#[4568F9BCA17391B], state=RUNNABLE,
group=TGRP-TestInputIterator]
[junit4] 2 at
org.apache.lucene.search.suggest.TestInputIterator.testTerms(TestInputIterator.java:143)
[junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
[junit4] 2 at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2 at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2 at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2 at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2 at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2 at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14648940#comment-14648940
]
Michael McCandless commented on LUCENE-6629:
Woops, you're right [~dawid.weiss]! It also stalls for me ... so this is just
SimpleText being slow (as expected).
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
Attachments: 54457_consoleText.txt
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14648919#comment-14648919
]
Dawid Weiss commented on LUCENE-6629:
-
To me it looks like it's just churning a lot of data and using simple text
codec? I tried running on of the repros locally and it also stalls.
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
Attachments: 54457_consoleText.txt
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14648195#comment-14648195
]
Michael McCandless commented on LUCENE-6629:
Maybe another example:
https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/915/console
{noformat}
Suite: org.apache.lucene.codecs.simpletext.TestSimpleTextTermVectorsFormat
[junit4] 2 jul 30, 2015 2:46:47 PM
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.codecs.simpletext.TestSimpleTextTermVectorsFormat
[junit4] 21) Thread[id=147,
name=TEST-TestSimpleTextTermVectorsFormat.testRamBytesUsed-seed#[4AF0D6F4E5896BB7],
state=RUNNABLE, group=TGRP-TestSimpleTextTermVectorsFormat]
[junit4] 2 at
org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:132)
[junit4] 2 at
org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:60)
[junit4] 2 at
org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsReader.readLine(SimpleTextTermVectorsReader.java:234)
[junit4] 2 at
org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsReader.get(SimpleTextTermVectorsReader.java:141)
[junit4] 2 at
org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:194)
[junit4] 2 at
org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:187)
[junit4] 2 at
org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:127)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4089)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3664)
[junit4] 2 at
org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1929)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4731)
[junit4] 2 at
org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:695)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4757)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4748)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1476)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1254)
[junit4] 2 at
org.apache.lucene.index.BaseIndexFileFormatTestCase.testRamBytesUsed(BaseIndexFileFormatTestCase.java:260)
[junit4] 2 at
org.apache.lucene.index.BaseTermVectorsFormatTestCase.testRamBytesUsed(BaseTermVectorsFormatTestCase.java:74)
[junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:606)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
[junit4] 2 at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2 at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2 at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2 at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2 at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2 at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14621227#comment-14621227
]
Michael McCandless commented on LUCENE-6629:
bq. Can we have the 1.9ea JVM upgraded to b71 on that machine? Maybe it's a JVM
issue that's been solved already.
The hangs seem to happen with 1.8.0_40 and 1.9ea...
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
Attachments: 54457_consoleText.txt
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14621225#comment-14621225
]
Michael McCandless commented on LUCENE-6629:
Another confusing build hang:
http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/55047/
{noformat}
[junit4] 2 NOTE: reproduce with: ant test -Dtestcase=TestJapaneseAnalyzer
-Dtests.method=test5thCuriousString -Dtests.seed=4C20E5ADD6D736D1
-Dtests.slow=true -Dtests.locale=pl_PL -Dtests.timezone=Indian/Mahe
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 7197s J2 | TestJapaneseAnalyzer.test5thCuriousString
[junit4] Throwable #1: java.lang.Exception: Test abandoned because
suite timeout was reached.
[junit4]at
__randomizedtesting.SeedInfo.seed([4C20E5ADD6D736D1]:0)
[junit4] 2 lip 09, 2015 5:49:51 PM
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2 WARNING: Will linger awaiting termination of 1 leaked
thread(s).
[junit4] 2 lip 09, 2015 5:50:11 PM
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2 SEVERE: 1 thread leaked from SUITE scope at
org.apache.lucene.analysis.ja.TestJapaneseAnalyzer:
[junit4] 21) Thread[id=13,
name=TEST-TestJapaneseAnalyzer.test5thCuriousString-seed#[4C20E5ADD6D736D1],
state=RUNNABLE, group=TGRP-TestJapaneseAnalyzer]
[junit4] 2 at
org.apache.lucene.analysis.ja.JapaneseTokenizer.backtrace(JapaneseTokenizer.java:1056)
[junit4] 2 at
org.apache.lucene.analysis.ja.JapaneseTokenizer.parse(JapaneseTokenizer.java:647)
[junit4] 2 at
org.apache.lucene.analysis.ja.JapaneseTokenizer.incrementToken(JapaneseTokenizer.java:474)
[junit4] 2 at
org.apache.lucene.analysis.ja.JapaneseBaseFormFilter.incrementToken(JapaneseBaseFormFilter.java:50)
[junit4] 2 at
org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51)
[junit4] 2 at
org.apache.lucene.analysis.cjk.CJKWidthFilter.incrementToken(CJKWidthFilter.java:63)
[junit4] 2 at
org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51)
[junit4] 2 at
org.apache.lucene.analysis.ja.JapaneseKatakanaStemFilter.incrementToken(JapaneseKatakanaStemFilter.java:63)
[junit4] 2 at
org.apache.lucene.analysis.core.LowerCaseFilter.incrementToken(LowerCaseFilter.java:45)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:706)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:680)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:676)
[junit4] 2 at
org.apache.lucene.analysis.ja.TestJapaneseAnalyzer.test5thCuriousString(TestJapaneseAnalyzer.java:217)
[junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
[junit4] 2 at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2 at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2 at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2 at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2 at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2 at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14618825#comment-14618825
]
Dawid Weiss commented on LUCENE-6629:
-
Can we have the 1.9ea JVM upgraded to b71 on that machine? Maybe it's a JVM
issue that's been solved already.
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
Attachments: 54457_consoleText.txt
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14618922#comment-14618922
]
Uwe Schindler commented on LUCENE-6629:
---
Policeman Jenkins has Java 9 EA b71 already.
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
Attachments: 54457_consoleText.txt
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14606016#comment-14606016
]
Robert Muir commented on LUCENE-6629:
-
I saw that, but its on a 1.9EA build and we have encountered some arraycopy
issues there? I don't think we should worry too much on those until we have a
more recent build in jenkins.
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14605324#comment-14605324
]
Michael McCandless commented on LUCENE-6629:
Here's another one (just happened last night):
http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/53453/console
{noformat}
[junit4] HEARTBEAT J2 PID(14791@localhost): 2015-06-29T01:59:23, stalled for
7144s at: TestIndexWriterCommit.testCommitUserData
[junit4] Suite: org.apache.lucene.index.TestIndexWriterCommit
[junit4] 2 6? 28, 2015 6:00:17 ??
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.index.TestIndexWriterCommit
[junit4] 21) Thread[id=12, name=JUnit4-serializer-daemon,
state=TIMED_WAITING, group=main]
[junit4] 2 at java.lang.Thread.sleep(Native Method)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
[junit4] 22) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1249)
[junit4] 2 at java.lang.Thread.join(Thread.java:1323)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 23) Thread[id=18,
name=SUITE-TestIndexWriterCommit-seed#[9DCD05E439A5112E], state=RUNNABLE,
group=TGRP-TestIndexWriterCommit]
[junit4] 2 at java.lang.Thread.getStackTrace(Thread.java:1556)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62)
[junit4] 2 at
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568)
[junit4] 24) Thread[id=19,
name=TEST-TestIndexWriterCommit.testCommitUserData-seed#[9DCD05E439A5112E],
state=RUNNABLE, group=TGRP-TestIndexWriterCommit]
[junit4] 2 at
org.apache.lucene.store.DataOutput.writeString(DataOutput.java:259)
[junit4] 2 at
org.apache.lucene.codecs.CodecUtil.writeHeader(CodecUtil.java:92)
[junit4] 2 at
org.apache.lucene.codecs.CodecUtil.writeIndexHeader(CodecUtil.java:134)
[junit4] 2 at
org.apache.lucene.codecs.lucene50.Lucene50SegmentInfoFormat.write(Lucene50SegmentInfoFormat.java:129)
[junit4] 2 at
org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:499)
[junit4] 2 at
org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:457)
[junit4] 2 at
org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:514)
[junit4] 2 at
org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:384)
[junit4] 2 at
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:477)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1470)
[junit4] 2 at
org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1249)
[junit4] 2 at
org.apache.lucene.index.TestIndexWriter.addDoc(TestIndexWriter.java:148)
[junit4] 2 at
org.apache.lucene.index.TestIndexWriterCommit.testCommitUserData(TestIndexWriterCommit.java:664)
[junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:502)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14604651#comment-14604651
]
Michael McCandless commented on LUCENE-6629:
Here's another possible case, from
http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/12746 (but now deleted):
{noformat}
[junit4] 21) Thread[id=149,
name=TEST-TestSnowballVocab.testStemmers-seed#[7FBAD445ED608074],
state=RUNNABLE, group=TGRP-TestSnowballVocab]
[junit4] 2 at
org.tartarus.snowball.ext.FrenchStemmer.stem(FrenchStemmer.java:1379)
[junit4] 2 at
org.apache.lucene.analysis.snowball.SnowballFilter.incrementToken(SnowballFilter.java:95)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:400)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:352)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:362)
[junit4] 2 at
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkOneTerm(BaseTokenStreamTestCase.java:423)
[junit4] 2 at
org.apache.lucene.analysis.VocabularyAssert.assertVocabulary(VocabularyAssert.java:47)
[junit4] 2 at
org.apache.lucene.analysis.VocabularyAssert.assertVocabulary(VocabularyAssert.java:73)
[junit4] 2 at
org.apache.lucene.analysis.snowball.TestSnowballVocab.assertCorrectOutput(TestSnowballVocab.java:80)
[junit4] 2 at
org.apache.lucene.analysis.snowball.TestSnowballVocab.testStemmers(TestSnowballVocab.java:45)
{noformat}
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[
https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14604653#comment-14604653
]
Robert Muir commented on LUCENE-6629:
-
I don't think that one is us. Those vocabulary asserts just stem the same .zip
files always. its deterministic and a rather small (maybe 100k) amount of
strings, same ones tested every time.
Random 7200 seconds build timeouts / infinite loops in Lucene tests?
Key: LUCENE-6629
URL: https://issues.apache.org/jira/browse/LUCENE-6629
Project: Lucene - Core
Issue Type: Bug
Reporter: Michael McCandless
I'm not sure what's going on here, but every so often a Jenkins run will fail
with a build timeout (7200 seconds) with stack traces that do not look like
deadlock. They never reproduce for me.
We really need to improve test infra here, so that each HEARTBEAT we also got
1) full thread stacks and 2) total CPU usage of the process as reported by
the ManagementBean APIs ... this would shed more light on whether the JVM is
somehow hung vs our bug (infinite loop). But infinite loop seems most likely
... the stacks always seem to be somewhere spooky.
We should try to gather recent Jenkins runs where this is happening, here, to
see if there are patterns / we can get to the root cause.
Anyway, this happened to me on my old beast box, which runs the nightly ant
test times graphs:
http://people.apache.org/~mikemccand/lucenebench/antcleantest.html
The 2015/06/27 data point is missing because it failed with this timeout:
{noformat}
[junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 2 ??? 28, 2015 7:01:29 ?
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2 WARNING: Suite execution timed out:
org.apache.lucene.search.TestDocValuesRewriteMethod
[junit4] 21) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2 at java.lang.Object.wait(Native Method)
[junit4] 2 at java.lang.Thread.join(Thread.java:1245)
[junit4] 2 at java.lang.Thread.join(Thread.java:1319)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
[junit4] 2 at
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
[junit4] 2 at
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
[junit4] 22) Thread[id=213,
name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8],
state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod]
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900)
[junit4] 2 at
org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389)
[junit4] 2 at
org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495)
[junit4] 2 at
org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109)
[junit4] 2 at
org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117)
[junit4] 2 at
org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2 at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497)
[junit4] 2 at
