Repository: drill Updated Branches: refs/heads/master 8d7458958 -> e85cdc54c
DRILL-3307: Query with window function runs out of memory Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e85cdc54 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e85cdc54 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e85cdc54 Branch: refs/heads/master Commit: e85cdc54cc21e0be2bf7169868ec00b8ef7c1f22 Parents: 8d74589 Author: Steven Phillips <[email protected]> Authored: Fri Jun 26 16:15:21 2015 -0700 Committer: Aman Sinha <[email protected]> Committed: Sat Jun 27 11:58:46 2015 -0700 ---------------------------------------------------------------------- .../exec/physical/impl/xsort/ExternalSortBatch.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/e85cdc54/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java index 02a1c08..6da5582 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java @@ -406,7 +406,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> { batchGroups.addAll(spilledBatchGroups); logger.warn("Starting to merge. {} batch groups. Current allocated memory: {}", batchGroups.size(), oContext.getAllocator().getAllocatedMemory()); VectorContainer hyperBatch = constructHyperBatch(batchGroups); - createCopier(hyperBatch, batchGroups, container); + createCopier(hyperBatch, batchGroups, container, false); int estimatedRecordSize = 0; for (VectorWrapper w : batchGroups.get(0)) { @@ -474,7 +474,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> { } int targetRecordCount = Math.max(1, 250 * 1000 / estimatedRecordSize); VectorContainer hyperBatch = constructHyperBatch(batchGroupList); - createCopier(hyperBatch, batchGroupList, outputContainer); + createCopier(hyperBatch, batchGroupList, outputContainer, true); int count = copier.next(targetRecordCount); assert count > 0; @@ -673,7 +673,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> { g.getEvalBlock()._return(JExpr.lit(0)); } - private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList, VectorContainer outputContainer) throws SchemaChangeException { + private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList, VectorContainer outputContainer, boolean spilling) throws SchemaChangeException { try { if (copier == null) { CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION, context.getFunctionRegistry()); @@ -689,11 +689,12 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> { copier.cleanup(); } + BufferAllocator allocator = spilling ? copierAllocator : oContext.getAllocator(); for (VectorWrapper<?> i : batch) { - ValueVector v = TypeHelper.getNewVector(i.getField(), copierAllocator); + ValueVector v = TypeHelper.getNewVector(i.getField(), allocator); outputContainer.add(v); } - copier.setup(context, copierAllocator, batch, batchGroupList, outputContainer); + copier.setup(context, allocator, batch, batchGroupList, outputContainer); } catch (ClassTransformationException e) { throw new RuntimeException(e); } catch (IOException e) {
