madrob commented on a change in pull request #1606: URL: https://github.com/apache/lucene-solr/pull/1606#discussion_r444334278
########## File path: solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java ########## @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.util.HashMap; +import java.util.Map; + +/** + * Manages all registered circuit breaker instances. Responsible for a holistic view + * of whether a circuit breaker has tripped or not. + * + * There are two typical ways of using this class's instance: + * 1. Check if any circuit breaker has triggered -- and know which circuit breaker has triggered. + * 2. Get an instance of a specific circuit breaker and perform checks. + * + * It is a good practise to register new circuit breakers here if you want them checked for every + * request. + */ +public class CircuitBreakerManager { + + private final Map<CircuitBreakerType, CircuitBreaker> circuitBreakerMap = new HashMap<>(); + + // Allows replacing of existing circuit breaker + public void registerCircuitBreaker(CircuitBreakerType circuitBreakerType, CircuitBreaker circuitBreaker) { + assert circuitBreakerType != null && circuitBreaker != null; + + circuitBreakerMap.put(circuitBreakerType, circuitBreaker); + } + + public CircuitBreaker getCircuitBreaker(CircuitBreakerType circuitBreakerType) { + assert circuitBreakerType != null; + + return circuitBreakerMap.get(circuitBreakerType); + } + + /** + * Check if any circuit breaker has triggered. + * @return CircuitBreakers which have triggered, null otherwise + */ + public Map<CircuitBreakerType, CircuitBreaker> checkAllCircuitBreakers() { + Map<CircuitBreakerType, CircuitBreaker> triggeredCircuitBreakers = new HashMap<>(); + + for (CircuitBreakerType circuitBreakerType : circuitBreakerMap.keySet()) { Review comment: prefer `entrySet` ########## File path: solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +import org.apache.solr.core.SolrCore; + +public class MemoryCircuitBreaker extends CircuitBreaker { + private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean(); + + // Assumption -- the value of these parameters will be set correctly before invoking printDebugInfo() + private double seenMemory; + private double allowedMemory; + + public MemoryCircuitBreaker(SolrCore solrCore) { + super(solrCore); + } + + // TODO: An optimization can be to trip the circuit breaker for a duration of time + // after the circuit breaker condition is matched. This will optimize for per call + // overhead of calculating the condition parameters but can result in false positives. + @Override + public boolean isCircuitBreakerGauntletTripped() { + if (!isCircuitBreakerEnabled()) { + return false; + } + + allowedMemory = getCurrentMemoryThreshold(); + + if (allowedMemory < 0) { + // No threshold + return false; + } + + seenMemory = calculateLiveMemoryUsage(); + + return (seenMemory >= allowedMemory); + } + + @Override + public String printDebugInfo() { + return "seen memory " + seenMemory + " allowed memory " + allowedMemory; Review comment: this would be better as seenMemory=X allowedMemory=Y - a little bit less human readable but a lot easier to grep for or search in something like Splunk. ########## File path: solr/core/src/java/org/apache/solr/core/SolrConfig.java ########## @@ -224,6 +224,9 @@ private SolrConfig(SolrResourceLoader loader, String name, boolean isConfigsetTr queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1)); queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE); enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false); + + useCircuitBreakers = getBool("query/useCircuitBreakers", false); + memoryCircuitBreakerThreshold = getInt("query/memoryCircuitBreakerThreshold", 100); Review comment: Should we validate that this is between 0 and 100? ########## File path: solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml ########## @@ -70,6 +70,10 @@ <queryResultWindowSize>10</queryResultWindowSize> + <useCircuitBreakers>false</useCircuitBreakers> + + <memoryCircuitBreakerThreshold>100</memoryCircuitBreakerThreshold> Review comment: Does this have to be set when useCircuitBreakers is false? I would like our configs to be more tolerant. ########## File path: solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +import org.apache.solr.core.SolrCore; + +public class MemoryCircuitBreaker extends CircuitBreaker { + private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean(); + + // Assumption -- the value of these parameters will be set correctly before invoking printDebugInfo() + private double seenMemory; + private double allowedMemory; + + public MemoryCircuitBreaker(SolrCore solrCore) { + super(solrCore); + } + + // TODO: An optimization can be to trip the circuit breaker for a duration of time + // after the circuit breaker condition is matched. This will optimize for per call + // overhead of calculating the condition parameters but can result in false positives. + @Override + public boolean isCircuitBreakerGauntletTripped() { + if (!isCircuitBreakerEnabled()) { + return false; + } + + allowedMemory = getCurrentMemoryThreshold(); + + if (allowedMemory < 0) { + // No threshold + return false; + } + + seenMemory = calculateLiveMemoryUsage(); + + return (seenMemory >= allowedMemory); + } + + @Override + public String printDebugInfo() { + return "seen memory " + seenMemory + " allowed memory " + allowedMemory; + } + + private double getCurrentMemoryThreshold() { + int thresholdValueInPercentage = solrCore.getSolrConfig().memoryCircuitBreakerThreshold; + long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax(); + + if (currentMaxHeap <= 0) { + return Long.MIN_VALUE; + } + + double thresholdInFraction = (double) thresholdValueInPercentage / 100; + double actualLimit = currentMaxHeap * thresholdInFraction; + + if (actualLimit <= 0) { + throw new IllegalStateException("Memory limit cannot be less than or equal to zero"); Review comment: Is this checking for overflow? How does the error condition occur otherwise? ########## File path: solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +import org.apache.solr.core.SolrCore; + +public class MemoryCircuitBreaker extends CircuitBreaker { + private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean(); + + // Assumption -- the value of these parameters will be set correctly before invoking printDebugInfo() + private double seenMemory; + private double allowedMemory; + + public MemoryCircuitBreaker(SolrCore solrCore) { + super(solrCore); + } + + // TODO: An optimization can be to trip the circuit breaker for a duration of time + // after the circuit breaker condition is matched. This will optimize for per call + // overhead of calculating the condition parameters but can result in false positives. + @Override + public boolean isCircuitBreakerGauntletTripped() { + if (!isCircuitBreakerEnabled()) { + return false; + } + + allowedMemory = getCurrentMemoryThreshold(); + + if (allowedMemory < 0) { + // No threshold + return false; + } + + seenMemory = calculateLiveMemoryUsage(); + + return (seenMemory >= allowedMemory); + } + + @Override + public String printDebugInfo() { + return "seen memory " + seenMemory + " allowed memory " + allowedMemory; + } + + private double getCurrentMemoryThreshold() { + int thresholdValueInPercentage = solrCore.getSolrConfig().memoryCircuitBreakerThreshold; + long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax(); + + if (currentMaxHeap <= 0) { + return Long.MIN_VALUE; + } + + double thresholdInFraction = (double) thresholdValueInPercentage / 100; + double actualLimit = currentMaxHeap * thresholdInFraction; Review comment: I think this makes more sense to be a long. Returning fractional bytes is meaningless. ########## File path: solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.util.NamedThreadFactory; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.CircuitBreakerType; +import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class TestCircuitBreaker extends SolrTestCaseJ4 { + private final static int NUM_DOCS = 20; + private static ExecutorService executor; + + @BeforeClass + public static void setUpClass() throws Exception { + System.setProperty("filterCache.enabled", "false"); + System.setProperty("queryResultCache.enabled", "false"); + System.setProperty("documentCache.enabled", "true"); + + executor = ExecutorUtil.newMDCAwareCachedThreadPool( + new SolrNamedThreadFactory("TestCircuitBreaker")); + initCore("solrconfig-memory-circuitbreaker.xml", "schema.xml"); + for (int i = 0 ; i < NUM_DOCS ; i ++) { + assertU(adoc("name", "john smith", "id", "1")); + assertU(adoc("name", "johathon smith", "id", "2")); + assertU(adoc("name", "john percival smith", "id", "3")); + assertU(commit()); + assertU(optimize()); + + //commit inside the loop to get multiple segments to make search as realistic as possible + assertU(commit()); + } + } + + @Override + public void tearDown() throws Exception { + executor = null; Review comment: Probably want to also call executor.shutdown in addition to null? ########## File path: solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.util.NamedThreadFactory; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.CircuitBreakerType; +import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class TestCircuitBreaker extends SolrTestCaseJ4 { + private final static int NUM_DOCS = 20; + private static ExecutorService executor; + + @BeforeClass + public static void setUpClass() throws Exception { + System.setProperty("filterCache.enabled", "false"); + System.setProperty("queryResultCache.enabled", "false"); + System.setProperty("documentCache.enabled", "true"); + + executor = ExecutorUtil.newMDCAwareCachedThreadPool( + new SolrNamedThreadFactory("TestCircuitBreaker")); + initCore("solrconfig-memory-circuitbreaker.xml", "schema.xml"); + for (int i = 0 ; i < NUM_DOCS ; i ++) { + assertU(adoc("name", "john smith", "id", "1")); + assertU(adoc("name", "johathon smith", "id", "2")); + assertU(adoc("name", "john percival smith", "id", "3")); + assertU(commit()); + assertU(optimize()); + + //commit inside the loop to get multiple segments to make search as realistic as possible + assertU(commit()); + } + } + + @Override + public void tearDown() throws Exception { + executor = null; + super.tearDown(); + } + + @AfterClass + public static void afterClass() { + System.clearProperty("filterCache.enabled"); + System.clearProperty("queryResultCache.enabled"); + System.clearProperty("documentCache.enabled"); + } + + public void testCBAlwaysTrips() throws IOException { + HashMap<String, String> args = new HashMap<String, String>(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + CircuitBreaker circuitBreaker = new MockCircuitBreaker(h.getCore()); + + h.getCore().getCircuitBreakerManager().registerCircuitBreaker(CircuitBreakerType.MEMORY, circuitBreaker); + + expectThrows(SolrException.class, () -> { + h.query(req("name:\"john smith\"")); + }); + } + + public void testCBFakeMemoryPressure() throws IOException { + HashMap<String, String> args = new HashMap<String, String>(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + CircuitBreaker circuitBreaker = new FakeMemoryPressureCircuitBreaker(h.getCore()); + + h.getCore().getCircuitBreakerManager().registerCircuitBreaker(CircuitBreakerType.MEMORY, circuitBreaker); + + expectThrows(SolrException.class, () -> { + h.query(req("name:\"john smith\"")); + }); + } + + public void testBuildingMemoryPressure() throws Exception { + HashMap<String, String> args = new HashMap<String, String>(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + AtomicInteger failureCount = new AtomicInteger(); + + CircuitBreaker circuitBreaker = new BuildingUpMemoryPressureCircuitBreaker(h.getCore()); + + h.getCore().getCircuitBreakerManager().registerCircuitBreaker(CircuitBreakerType.MEMORY, circuitBreaker); + + for (int i = 0; i < 5; i++) { + System.out.println("i is " + i); Review comment: Use logger instead of system.out ########## File path: solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +import org.apache.solr.core.SolrCore; + +public class MemoryCircuitBreaker extends CircuitBreaker { + private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean(); + + // Assumption -- the value of these parameters will be set correctly before invoking printDebugInfo() + private double seenMemory; + private double allowedMemory; + + public MemoryCircuitBreaker(SolrCore solrCore) { + super(solrCore); + } + + // TODO: An optimization can be to trip the circuit breaker for a duration of time + // after the circuit breaker condition is matched. This will optimize for per call + // overhead of calculating the condition parameters but can result in false positives. + @Override + public boolean isCircuitBreakerGauntletTripped() { + if (!isCircuitBreakerEnabled()) { + return false; + } + + allowedMemory = getCurrentMemoryThreshold(); + + if (allowedMemory < 0) { + // No threshold + return false; + } + + seenMemory = calculateLiveMemoryUsage(); + + return (seenMemory >= allowedMemory); + } + + @Override + public String printDebugInfo() { + return "seen memory " + seenMemory + " allowed memory " + allowedMemory; + } + + private double getCurrentMemoryThreshold() { + int thresholdValueInPercentage = solrCore.getSolrConfig().memoryCircuitBreakerThreshold; + long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax(); Review comment: This should be constant, right? Get it once at constructor? ########## File path: solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.util.NamedThreadFactory; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.CircuitBreakerType; +import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class TestCircuitBreaker extends SolrTestCaseJ4 { + private final static int NUM_DOCS = 20; + private static ExecutorService executor; + + @BeforeClass + public static void setUpClass() throws Exception { + System.setProperty("filterCache.enabled", "false"); + System.setProperty("queryResultCache.enabled", "false"); + System.setProperty("documentCache.enabled", "true"); + + executor = ExecutorUtil.newMDCAwareCachedThreadPool( + new SolrNamedThreadFactory("TestCircuitBreaker")); + initCore("solrconfig-memory-circuitbreaker.xml", "schema.xml"); + for (int i = 0 ; i < NUM_DOCS ; i ++) { + assertU(adoc("name", "john smith", "id", "1")); + assertU(adoc("name", "johathon smith", "id", "2")); + assertU(adoc("name", "john percival smith", "id", "3")); + assertU(commit()); + assertU(optimize()); + + //commit inside the loop to get multiple segments to make search as realistic as possible + assertU(commit()); Review comment: commit-optimize-commit in a loop won't get you multiple segments ########## File path: solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +import org.apache.solr.core.SolrCore; + +public class MemoryCircuitBreaker extends CircuitBreaker { + private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean(); + + // Assumption -- the value of these parameters will be set correctly before invoking printDebugInfo() + private double seenMemory; + private double allowedMemory; + + public MemoryCircuitBreaker(SolrCore solrCore) { + super(solrCore); + } + + // TODO: An optimization can be to trip the circuit breaker for a duration of time + // after the circuit breaker condition is matched. This will optimize for per call + // overhead of calculating the condition parameters but can result in false positives. + @Override + public boolean isCircuitBreakerGauntletTripped() { + if (!isCircuitBreakerEnabled()) { + return false; + } + + allowedMemory = getCurrentMemoryThreshold(); + + if (allowedMemory < 0) { + // No threshold + return false; + } + + seenMemory = calculateLiveMemoryUsage(); + + return (seenMemory >= allowedMemory); + } + + @Override + public String printDebugInfo() { + return "seen memory " + seenMemory + " allowed memory " + allowedMemory; + } + + private double getCurrentMemoryThreshold() { + int thresholdValueInPercentage = solrCore.getSolrConfig().memoryCircuitBreakerThreshold; + long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax(); + + if (currentMaxHeap <= 0) { + return Long.MIN_VALUE; + } + + double thresholdInFraction = (double) thresholdValueInPercentage / 100; + double actualLimit = currentMaxHeap * thresholdInFraction; + + if (actualLimit <= 0) { + throw new IllegalStateException("Memory limit cannot be less than or equal to zero"); + } + + return actualLimit; + } + + /** + * Calculate the live memory usage for the system. This method has package visibility + * to allow using for testing + * @return Memory usage in bytes + */ + protected long calculateLiveMemoryUsage() { + return MEMORY_MX_BEAN.getHeapMemoryUsage().getUsed(); Review comment: Is this expensive to get? If we're doing it for every query, I'd want to make sure that it is an efficient call. ########## File path: solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.util.NamedThreadFactory; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.CircuitBreakerType; +import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class TestCircuitBreaker extends SolrTestCaseJ4 { + private final static int NUM_DOCS = 20; + private static ExecutorService executor; + + @BeforeClass + public static void setUpClass() throws Exception { + System.setProperty("filterCache.enabled", "false"); + System.setProperty("queryResultCache.enabled", "false"); + System.setProperty("documentCache.enabled", "true"); + + executor = ExecutorUtil.newMDCAwareCachedThreadPool( + new SolrNamedThreadFactory("TestCircuitBreaker")); + initCore("solrconfig-memory-circuitbreaker.xml", "schema.xml"); + for (int i = 0 ; i < NUM_DOCS ; i ++) { + assertU(adoc("name", "john smith", "id", "1")); + assertU(adoc("name", "johathon smith", "id", "2")); + assertU(adoc("name", "john percival smith", "id", "3")); + assertU(commit()); + assertU(optimize()); + + //commit inside the loop to get multiple segments to make search as realistic as possible + assertU(commit()); + } + } + + @Override + public void tearDown() throws Exception { + executor = null; + super.tearDown(); + } + + @AfterClass + public static void afterClass() { + System.clearProperty("filterCache.enabled"); + System.clearProperty("queryResultCache.enabled"); + System.clearProperty("documentCache.enabled"); + } + + public void testCBAlwaysTrips() throws IOException { + HashMap<String, String> args = new HashMap<String, String>(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + CircuitBreaker circuitBreaker = new MockCircuitBreaker(h.getCore()); + + h.getCore().getCircuitBreakerManager().registerCircuitBreaker(CircuitBreakerType.MEMORY, circuitBreaker); + + expectThrows(SolrException.class, () -> { + h.query(req("name:\"john smith\"")); + }); + } + + public void testCBFakeMemoryPressure() throws IOException { + HashMap<String, String> args = new HashMap<String, String>(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + CircuitBreaker circuitBreaker = new FakeMemoryPressureCircuitBreaker(h.getCore()); + + h.getCore().getCircuitBreakerManager().registerCircuitBreaker(CircuitBreakerType.MEMORY, circuitBreaker); + + expectThrows(SolrException.class, () -> { + h.query(req("name:\"john smith\"")); + }); + } + + public void testBuildingMemoryPressure() throws Exception { + HashMap<String, String> args = new HashMap<String, String>(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + AtomicInteger failureCount = new AtomicInteger(); + + CircuitBreaker circuitBreaker = new BuildingUpMemoryPressureCircuitBreaker(h.getCore()); + + h.getCore().getCircuitBreakerManager().registerCircuitBreaker(CircuitBreakerType.MEMORY, circuitBreaker); + + for (int i = 0; i < 5; i++) { + System.out.println("i is " + i); + executor.submit(() -> { + try { + h.query(req("name:\"john smith\"")); + } catch (SolrException e) { + failureCount.incrementAndGet(); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + }); + } + + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e.getMessage()); + } + + assert failureCount.get() == 1; Review comment: use unit asserts instead of java asserts please ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
