[
https://issues.apache.org/jira/browse/SOLR-15819?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17448787#comment-17448787
]
Chris M. Hostetter commented on SOLR-15819:
-------------------------------------------
Sample failure from a recent jenkins run...
{noformat}
org.apache.solr.util.TestCircuitBreaker > testResponseWithCBTiming FAILED
java.lang.RuntimeException: Exception during query
at
__randomizedtesting.SeedInfo.seed([4229D84A41A9B403:AAF575A8B241AE23]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:1009)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:971)
at
org.apache.solr.util.TestCircuitBreaker.testResponseWithCBTiming(TestCircuitBreaker.java:247)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by:
org.apache.solr.common.SolrException: Circuit Breakers tripped Memory
Circuit Breaker triggered as JVM heap usage values are greater than allocated
threshold.Seen JVM heap memory usage 424442256 and allocated threshold 402653184
at
app//org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:323)
at
app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at app//org.apache.solr.core.SolrCore.execute(SolrCore.java:2639)
at app//org.apache.solr.util.TestHarness.query(TestHarness.java:346)
at app//org.apache.solr.util.TestHarness.query(TestHarness.java:327)
at
app//org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:985)
... 48 more
2> 1950458 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to
test-framework derived value of
'/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/server/solr/configsets/_default/conf'
2> 1950458 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
2> 1950459 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.u.ErrorLogMuter Closing ErrorLogMuter-regex-549 after mutting 0 log
messages
2> 1950459 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-550 for ERROR logs matching
regex: ignore_exception
2> 1950460 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCaseJ4 Created dataDir:
/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core/build/tmp/tests-tmp/solr.util.TestCircuitBreaker_4229D84A41A9B403-001/data-dir-228-001
2> 1950460 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
w/NUMERIC_DOCVALUES_SYSPROP=true
2> 1950461 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via:
@org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0,
clientAuth=0.0/0.0)
2> 1950461 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCaseJ4 ####initCore
2> 1950465 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.c.SolrResourceLoader Added 2 libs to classloader, from paths:
[/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core/build/resources/test/solr/collection1/lib,
/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-
main/solr/core/build/resources/test/solr/collection1/lib/classes]
2> 1950476 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
2> 1950476 WARN (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.c.SolrConfig solrconfig.xml: <boolTofilterOptimizer> is currently not
implemented and has no effect.
2> 1950479 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.s.IndexSchema Schema name=test
2> 1950485 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
2> 1951259 WARN (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.e.j.u.s.S.config Trusting all certificates configured for
Client@617bcf2e[provider=null,keyStore=null,trustStore=null]
2> 1951259 WARN (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
Client@617bcf2e[provider=null,keyStore=null,trustStore=null]
2> 1951261 WARN (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.e.j.u.s.S.config Trusting all certificates configured for
Client@f9f658f[provider=null,keyStore=null,trustStore=null]
2> 1951261 WARN (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
Client@f9f658f[provider=null,keyStore=null,trustStore=null]
2> 1951263 WARN (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.c.CoreContainer Not all security plugins configured!
authentication=disabled authorization=disabled. Solr is only as secure as you
make it. Consider configuring authentication/authorization before exposing Solr
to users interna
l or external. See https://s.apache.org/solrsecurity for more info
2> 1951277 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node')
enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@32e16297
2> 1951283 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm')
enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@32e16297
2> 1951283 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry
'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@32e16297
2> 1951284 INFO (coreLoadExecutor-17379-thread-1) []
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient core cache for max
2147483647 cores with initial capacity of 1024
2> 1951287 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.c.SolrResourceLoader Added 2 libs to classloader, from paths:
[/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core/build/resources/test/solr/collection1/lib,
/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/
core/build/resources/test/solr/collection1/lib/classes]
2> 1951299 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
2> 1951299 WARN (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.c.SolrConfig solrconfig.xml: <boolTofilterOptimizer> is currently not
implemented and has no effect.
2> 1951303 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.s.IndexSchema Schema name=test
2> 1951308 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
2> 1951315 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from
instancedir
/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core/build/resources/test/solr/collection1,
trusted=true
2> 1951315 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry
'solr.core.collection1') enabled at server:
com.sun.jmx.mbeanserver.JmxMBeanServer@32e16297
2> 1951315 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at
[/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core/build/resources/test/solr/collection1],
dataDir=[/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core
/build/tmp/tests-tmp/solr.util.TestCircuitBreaker_4229D84A41A9B403-001/data-dir-228-001/]
2> 1951316 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
maxMergeAtOnce=19, maxMergedSegmentMB=40.8876953125,
floorSegmentMB=0.4853515625, forceMergeDeletesPctAllowed=6.14008859683
6664, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12,
noCFSRatio=1.0, deletesPctAllowed=35.346987048354926
2> 1951346 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.u.CommitTracker Hard AutoCommit: disabled
2> 1951346 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.u.CommitTracker Soft AutoCommit: disabled
2> 1951346 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648,
maxMergeSizeForForcedMerge=9223372036854775807, ca
librateSizeByDeletes=false, maxMergeDocs=2147483647,
maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5505420790605812]
2> 1951347 WARN (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.r.ManagedResourceStorage Cannot write to config directory
/home/jenkins/jenkins-slave/workspace/Solr/Solr-Check-main/solr/core/build/resources/test/solr/collection1/conf
; switching to use InMemory storage instead.
2> 1951347 INFO (coreLoadExecutor-17379-thread-1) [ x:collection1]
o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2> 1951348 INFO (searcherExecutor-17381-thread-1-processing-collection1) [
x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher autowarm
time: 0 ms
2> 1951348 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) []
o.a.s.SolrTestCaseJ4 ####initCore end
2> 1951349 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) [
x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null
path=null params={}{add=[1]} 0 0
...
2> 1951495 INFO (searcherExecutor-17381-thread-1-processing-collection1) [
x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher autowarm
time: 0 ms
2> 1951496 INFO (SUITE-TestCircuitBreaker-seed#[4229D84A41A9B403]-worker) [
x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null
path=null params={}{commit=} 0 4
2> 1951500 INFO
(TEST-TestCircuitBreaker.testResponseWithCBTiming-seed#[4229D84A41A9B403]) []
o.a.s.SolrTestCaseJ4 ###Starting testResponseWithCBTiming
2> 1951501 INFO
(TEST-TestCircuitBreaker.testResponseWithCBTiming-seed#[4229D84A41A9B403]) []
o.a.s.c.S.Request [collection1] webapp=null path=null
params={q=*:*&wt=xml&debugQuery=true} status=503 QTime=0
2> 1951501 ERROR
(TEST-TestCircuitBreaker.testResponseWithCBTiming-seed#[4229D84A41A9B403]) []
o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=*:*&wt=xml&debugQuery=true
2> => org.apache.solr.common.SolrException: Circuit Breakers
tripped Memory Circuit Breaker triggered as JVM heap usage values are greater
than allocated threshold.Seen JVM heap memory usage 424442256 and allocated
threshold 402653184
2>
2> org.apache.solr.common.SolrException: Circuit Breakers tripped Memory
Circuit Breaker triggered as JVM heap usage values are greater than allocated
threshold.Seen JVM heap memory usage 424442256 and allocated threshold 402653184
...
2> NOTE: reproduce with: gradlew test --tests
TestCircuitBreaker.testResponseWithCBTiming -Dtests.seed=4229D84A41A9B403
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en-MO
-Dtests.timezone=Etc/GMT+7 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
{noformat}
> spurious failures from TestCircuitBreaker.testResponseWithCBTiming
> ------------------------------------------------------------------
>
> Key: SOLR-15819
> URL: https://issues.apache.org/jira/browse/SOLR-15819
> Project: Solr
> Issue Type: Bug
> Security Level: Public(Default Security Level. Issues are Public)
> Reporter: Chris M. Hostetter
> Priority: Major
>
> As seen in both jenkins runs and locally on my machine,
> {{TestCircuitBreaker.testResponseWithCBTiming}} can fail in non-reproducible
> ways due to "Circuit Breakers tripped Memory Circuit Breaker triggered as JVM
> heap usage values are greater than allocated threshold.Seen JVM heap memory
> usage 424442256 and allocated threshold 402653184"
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
