[ https://issues.apache.org/jira/browse/DRILL-5815?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16195477#comment-16195477 ]
ASF GitHub Bot commented on DRILL-5815: --------------------------------------- Github user Ben-Zvi commented on a diff in the pull request: https://github.com/apache/drill/pull/960#discussion_r143316843 --- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java --- @@ -56,30 +59,90 @@ public static void setupBufferedOpsMemoryAllocations(final PhysicalPlan plan, fi } // if there are any sorts, compute the maximum allocation, and set it on them - if (bufferedOpList.size() > 0) { - final OptionManager optionManager = queryContext.getOptions(); - double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE); - final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE); - final long maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth); - long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(), - queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC)); - maxAllocPerNode = Math.min(maxAllocPerNode, - optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val); - final long maxOperatorAlloc = maxAllocPerNode / (bufferedOpList.size() * maxWidthPerNode); - logger.debug("Max buffered operator alloc: {}", maxOperatorAlloc); - - // User configurable option to allow forcing minimum memory. - // Ensure that the buffered ops receive the minimum memory needed to make progress. - // Without this, the math might work out to allocate too little memory. - final long opMinMem = queryContext.getOptions().getOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY).num_val; - - for(final PhysicalOperator op : bufferedOpList) { - - long alloc = Math.max(maxOperatorAlloc, op.getInitialAllocation()); - alloc = Math.max(alloc, opMinMem); - op.setMaxAllocation(alloc); - } - } plan.getProperties().hasResourcePlan = true; + if (bufferedOpList.isEmpty()) { + return; + } + + // Setup options, etc. + + final OptionManager optionManager = queryContext.getOptions(); + final long directMemory = DrillConfig.getMaxDirectMemory(); + + // Compute per-node, per-query memory. + + final long maxAllocPerNode = computeQueryMemory(queryContext.getConfig(), optionManager, directMemory); + logger.debug("Memory per query per node: {}", maxAllocPerNode); --- End diff -- Created DRILL-5852 > Provide option to set query memory as percent of total > ------------------------------------------------------ > > Key: DRILL-5815 > URL: https://issues.apache.org/jira/browse/DRILL-5815 > Project: Apache Drill > Issue Type: Improvement > Affects Versions: 1.11.0 > Reporter: Paul Rogers > Assignee: Paul Rogers > Fix For: 1.12.0 > > > Drill provides a parameter to set the memory per query as a static number > which defaults to 2 GB. This number is a wonderful setting for the default > Drillbit configuration of 8 GB heap; it allows 2-3 concurrent queries. But, > as Drillbit memory increases, the default becomes a bit constraining. While > users can change the setting, they seldom do. > In addition, provide an option that sets memory as a percent of total memory. > If the allocation is 10%, say, and total memory is 128 GB, then each query > gets ~13GB, which is a big improvement. > The existing option acts as a floor: the query must receive at least that > much memory. -- This message was sent by Atlassian JIRA (v6.4.14#64029)