This is an automated email from the ASF dual-hosted git repository. bohdan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2aa5b385de22ddcee66cdb763d654364791731e1 Author: Kunal Khatua <[email protected]> AuthorDate: Thu Aug 8 12:34:12 2019 -0700 DRILL-7338: REST API calls to Drill fail due to insufficient heap memory This PR allows for the 85% threshold to be customizable with a value of 0 meant for disabling. closes #1837 --- .../src/main/java/org/apache/drill/exec/ExecConstants.java | 2 ++ .../main/java/org/apache/drill/exec/server/rest/QueryWrapper.java | 7 ++++--- exec/java-exec/src/main/resources/drill-module.conf | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 1815855..6f3f17d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -229,6 +229,8 @@ public final class ExecConstants { public static final String HTTP_WEB_CLIENT_RESULTSET_AUTOLIMIT_CHECKED = "drill.exec.http.web.client.resultset.autolimit.checked"; public static final String HTTP_WEB_CLIENT_RESULTSET_AUTOLIMIT_ROWS = "drill.exec.http.web.client.resultset.autolimit.rows"; public static final String HTTP_WEB_CLIENT_RESULTSET_ROWS_PER_PAGE_VALUES = "drill.exec.http.web.client.resultset.rowsPerPageValues"; + //Control Heap usage runaway + public static final String HTTP_MEMORY_HEAP_FAILURE_THRESHOLD = "drill.exec.http.memory.heap.failure.threshold"; //Customize filters in options public static final String HTTP_WEB_OPTIONS_FILTERS = "drill.exec.http.web.options.filters"; public static final String SYS_STORE_PROVIDER_CLASS = "drill.exec.sys.store.provider.class"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java index 43dd737..5a4ce80 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java @@ -43,8 +43,6 @@ import java.util.concurrent.TimeUnit; @XmlRootElement public class QueryWrapper { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryWrapper.class); - // Heap usage threshold/trigger to provide resiliency on web server for queries submitted via HTTP - private static final double HEAP_MEMORY_FAILURE_THRESHOLD = 0.85; private final String query; private final String queryType; @@ -87,6 +85,9 @@ public class QueryWrapper { } webUserConnection.setAutoLimitRowCount(maxRows); + // Heap usage threshold/trigger to provide resiliency on web server for queries submitted via HTTP + double memoryFailureThreshold = workManager.getContext().getConfig().getDouble(ExecConstants.HTTP_MEMORY_HEAP_FAILURE_THRESHOLD); + // Submit user query to Drillbit work queue. final QueryId queryId = workManager.getUserWorker().submitWork(webUserConnection, runQuery); @@ -101,7 +102,7 @@ public class QueryWrapper { isComplete = webUserConnection.await(TimeUnit.SECONDS.toMillis(1)); //periodically timeout 1 sec to check heap } catch (InterruptedException e) {} usagePercent = getHeapUsage(); - if (usagePercent > HEAP_MEMORY_FAILURE_THRESHOLD) { + if (memoryFailureThreshold > 0 && usagePercent > memoryFailureThreshold) { nearlyOutOfHeapSpace = true; } } while (!isComplete && !nearlyOutOfHeapSpace); diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index dbd73ce..bb8f292 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -168,6 +168,7 @@ drill.exec: { maximum: 9223372036854775807 } }, + memory.heap.failure.threshold: 0.85, web.client.resultset: { autolimit { checked: false,
