Repository: tajo Updated Branches: refs/heads/branch-0.11.1 0e8c82a32 -> fcc2215fb
TAJO-2036: Prevent out of memory in the master server, if the query result is large. Signed-off-by: Jihoon Son <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fcc2215f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fcc2215f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fcc2215f Branch: refs/heads/branch-0.11.1 Commit: fcc2215fb63653d3e7c7a57725e35ff6f997b77f Parents: 0e8c82a Author: Byunghwa Yun <[email protected]> Authored: Wed Jan 27 22:29:15 2016 -0800 Committer: Jihoon Son <[email protected]> Committed: Wed Jan 27 22:29:38 2016 -0800 ---------------------------------------------------------------------- .../tajo/webapp/QueryExecutorServlet.java | 25 +++++---- .../resources/webapps/admin/query_executor.jsp | 54 +++++++++++--------- 2 files changed, 45 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc2215f/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index a04ff6c..2068ace 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -17,6 +17,7 @@ import org.apache.tajo.exception.TajoException; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.jdbc.FetchResultSet; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.util.Bytes; import org.apache.tajo.util.JSPUtil; import org.apache.tajo.util.TajoIdUtils; import org.codehaus.jackson.map.DeserializationConfig; @@ -161,6 +162,11 @@ public class QueryExecutorServlet extends HttpServlet { } catch (java.lang.NumberFormatException nfe) { queryRunner.sizeLimit = 1048576; } + try { + queryRunner.rowLimit = Integer.parseInt(request.getParameter("limitRow")); + } catch (java.lang.NumberFormatException nfe) { + queryRunner.rowLimit = 3000000; + } synchronized(queryRunners) { queryRunners.put(queryRunnerId, queryRunner); } @@ -197,7 +203,6 @@ public class QueryExecutorServlet extends HttpServlet { errorResponse(response, queryRunner.error); return; } - returnValue.put("numOfRows", queryRunner.numOfRows); returnValue.put("resultSize", queryRunner.resultRows); returnValue.put("resultData", queryRunner.queryResult); returnValue.put("resultColumns", queryRunner.columnNames); @@ -286,7 +291,7 @@ public class QueryExecutorServlet extends HttpServlet { String database; long resultRows; int sizeLimit; - long numOfRows; + long rowLimit; Exception error; AtomicInteger progress = new AtomicInteger(0); @@ -500,21 +505,21 @@ public class QueryExecutorServlet extends HttpServlet { columnNames.add(rsmd.getColumnName(i + 1)); } queryResult = new ArrayList<List<Object>>(); - - if(sizeLimit < resultRows) { - numOfRows = (long)((float)(resultRows) * ((float)sizeLimit / (float) resultRows)); - } else { - numOfRows = resultRows; - } + int currentResultSize = 0; int rowCount = 0; while (res.next()) { - if(rowCount > numOfRows) { + if(rowCount > rowLimit || currentResultSize > sizeLimit) { break; } List<Object> row = new ArrayList<Object>(); for(int i = 0; i < numOfColumns; i++) { - row.add(String.valueOf(res.getObject(i + 1))); + String columnValue = String.valueOf(res.getObject(i + 1)); + try { + currentResultSize += columnValue.getBytes(Bytes.UTF8_ENCODING).length; + } catch (Exception e) { + } + row.add(columnValue); } queryResult.add(row); rowCount++; http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc2215f/tajo-core/src/main/resources/webapps/admin/query_executor.jsp ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp index 6b6a3db..f359c99 100644 --- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp +++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp @@ -64,6 +64,7 @@ var progressTimer = null; var queryRunnerId = null; var PRINT_LIMIT = 25; var SIZE_LIMIT = 104857600; // Limit size of displayed results.(Bytes) +var ROW_LIMIT = 3000000; var pageNum = 0; var pageCount, storedColumns, storedData; @@ -87,6 +88,9 @@ function runQuery() { } else if(Math.ceil(Number($("#sizeLimit").val())) > 0) { SIZE_LIMIT = Number($("#sizeLimit").val()) * 1024 * 1024; } + if(Math.ceil(Number($("#rowLimit").val())) > 0) { + ROW_LIMIT = Number($("#rowLimit").val()) * 1000 * 1000; + } if(Math.ceil(Number($("#printLimit").val())) > 0) { PRINT_LIMIT = Number($("#printLimit").val()); } @@ -101,7 +105,7 @@ function runQuery() { $.ajax({ type: "POST", url: "query_exec", - data: { action: "runQuery", query: query, prevQueryId: queryRunnerId, limitSize:SIZE_LIMIT, database: sbox.options[sbox.selectedIndex].text } + data: { action: "runQuery", query: query, prevQueryId: queryRunnerId, limitSize:SIZE_LIMIT, limitRow:ROW_LIMIT, database: sbox.options[sbox.selectedIndex].text } }) .done(function(msg) { var resultJson = $.parseJSON(msg); @@ -234,26 +238,26 @@ function getCSV() { } function getNext() { - var printedLine = 0; - if(pageCount > pageNum) { - pageNum++; - document.getElementById("selectPage").options.selectedIndex = pageNum; - }else { - alert("There's no next page."); - return; - } - getPage(); + var printedLine = 0; + if(pageCount > pageNum) { + pageNum++; + document.getElementById("selectPage").options.selectedIndex = pageNum; + } else { + alert("There's no next page."); + return; + } + getPage(); } function getPrev() { - if(pageNum > 0 ) { - pageNum--; - document.getElementById("selectPage").options.selectedIndex = pageNum; - } else { - alert("There's no previous page."); - return; - } - getPage(); + if(pageNum > 0 ) { + pageNum--; + document.getElementById("selectPage").options.selectedIndex = pageNum; + } else { + alert("There's no previous page."); + return; + } + getPage(); } function getSelectedPage() { @@ -298,18 +302,20 @@ function getPage() { Database : <select id="selectDatabase" name="database" width="190" style="width: 190px"> <% - for (String databaseName : master.getCatalog().getAllDatabaseNames()) { - %> - <option value="<%=databaseName%>" <%= (databaseName.equals("default"))?"selected":"" %> ><%=databaseName%></option> - <% - } - %> + for (String databaseName : master.getCatalog().getAllDatabaseNames()) { + %> + <option value="<%=databaseName%>" <%= (databaseName.equals("default"))?"selected":"" %> ><%=databaseName%></option> + <% + } + %> </select> <p /> <textarea id="query" style="width:800px; height:250px; font-family:Tahoma; font-size:12px;"></textarea> <p /> Limit : <input id="sizeLimit" type="text" value="10" style="width:30px; text-align:center;" /> MB <p /> + Limit Rows : <input id="rowLimit" type="text" value="3" style="width:30px; text-align:center;" /> M + <p /> Rows/Page : <input id="printLimit" type="text" value="25" style="width:30px; text-align:center;" /> <hr /> <input id="btnSubmit" type="submit" value="Submit">
