[
https://issues.apache.org/jira/browse/DRILL-6477?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16511343#comment-16511343
]
ASF GitHub Bot commented on DRILL-6477:
---------------------------------------
ilooner closed pull request #1309: DRILL-6477: Drillbit crashes with OOME
(Heap) for a large WebUI query
URL: https://github.com/apache/drill/pull/1309
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 1070d762f9..776c469300 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -204,7 +204,6 @@ private ExecConstants() {
public static final String SERVICE_KEYTAB_LOCATION = SERVICE_LOGIN_PREFIX +
".keytab";
public static final String KERBEROS_NAME_MAPPING = SERVICE_LOGIN_PREFIX +
".auth_to_local";
-
public static final String USER_SSL_ENABLED =
"drill.exec.security.user.encryption.ssl.enabled";
public static final String BIT_ENCRYPTION_SASL_ENABLED =
"drill.exec.security.bit.encryption.sasl.enabled";
public static final String BIT_ENCRYPTION_SASL_MAX_WRAPPED_SIZE =
"drill.exec.security.bit.encryption.sasl.max_wrapped_size";
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 911ac0fb06..cf74937103 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -20,7 +20,10 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Maps;
+
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.proto.UserProtos.RunQuery;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
@@ -28,18 +31,26 @@
import org.apache.drill.exec.work.WorkManager;
import javax.xml.bind.annotation.XmlRootElement;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
@XmlRootElement
public class QueryWrapper {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(QueryWrapper.class);
+ // Heap usage threshold/trigger to provide resiliency on web server for
queries submitted via HTTP
+ private static final double HEAP_MEMORY_FAILURE_THRESHOLD = 0.85;
private final String query;
private final String queryType;
+ private static MemoryMXBean memMXBean = ManagementFactory.getMemoryMXBean();
+
@JsonCreator
public QueryWrapper(@JsonProperty("query") String query,
@JsonProperty("queryType") String queryType) {
this.query = query;
@@ -59,7 +70,6 @@ public QueryType getType() {
}
public QueryResult run(final WorkManager workManager, final
WebUserConnection webUserConnection) throws Exception {
-
final RunQuery runQuery = RunQuery.newBuilder().setType(getType())
.setPlan(getQuery())
.setResultsMode(QueryResultsMode.STREAM_FULL)
@@ -68,8 +78,37 @@ public QueryResult run(final WorkManager workManager, final
WebUserConnection we
// Submit user query to Drillbit work queue.
final QueryId queryId =
workManager.getUserWorker().submitWork(webUserConnection, runQuery);
+ boolean isComplete = false;
+ boolean nearlyOutOfHeapSpace = false;
+ float usagePercent = getHeapUsage();
+
// Wait until the query execution is complete or there is error submitting
the query
- webUserConnection.await();
+ logger.debug("Wait until the query execution is complete or there is error
submitting the query");
+ do {
+ try {
+ isComplete = webUserConnection.await(TimeUnit.SECONDS.toMillis(1));
/*periodically timeout to check heap*/
+ } catch (Exception e) { }
+
+ usagePercent = getHeapUsage();
+ if (usagePercent > HEAP_MEMORY_FAILURE_THRESHOLD) {
+ nearlyOutOfHeapSpace = true;
+ }
+ } while (!isComplete && !nearlyOutOfHeapSpace);
+
+ //Fail if nearly out of heap space
+ if (nearlyOutOfHeapSpace) {
+ workManager.getBee().getForemanForQueryId(queryId)
+ .addToEventQueue(QueryState.FAILED,
+ UserException.resourceError(
+ new Throwable(
+ "There is not enough heap memory to run this query using
the web interface. "
+ + "Please try a query with fewer columns or with a filter
or limit condition to limit the data returned. "
+ + "You can also try an ODBC/JDBC client. "
+ )
+ )
+ .build(logger)
+ );
+ }
if (logger.isTraceEnabled()) {
logger.trace("Query {} is completed ", queryId);
@@ -83,6 +122,11 @@ public QueryResult run(final WorkManager workManager, final
WebUserConnection we
return new QueryResult(queryId, webUserConnection.columns,
webUserConnection.results);
}
+ //Detect possible excess heap
+ private float getHeapUsage() {
+ return (float) memMXBean.getHeapMemoryUsage().getUsed() /
memMXBean.getHeapMemoryUsage().getMax();
+ }
+
public static class QueryResult {
private final String queryId;
public final Collection<String> columns;
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Drillbit hangs/crashes with OOME Java Heap Space for a large query through
> WebUI
> --------------------------------------------------------------------------------
>
> Key: DRILL-6477
> URL: https://issues.apache.org/jira/browse/DRILL-6477
> Project: Apache Drill
> Issue Type: Bug
> Components: Web Server
> Affects Versions: 1.13.0
> Reporter: Kunal Khatua
> Assignee: Kunal Khatua
> Priority: Major
> Labels: ready-to-commit
> Fix For: 1.14.0
>
>
> For queries submitted through the WebUI and retrieving a large resultset, the
> Drillbit often hangs or crashes due to the (foreman) Drillbit running out of
> Heap memory.
> This is because the Web client translates the resultset into a massive object
> in the heap-space and tries to send that back to the browser. This results in
> the VM thread actively trying to perform GC if the memory is not sufficient.
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
