http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java index e785f0d..098922c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java @@ -24,9 +24,9 @@ import io.netty.buffer.PooledByteBufAllocator; import io.netty.buffer.PooledByteBufAllocatorL; import java.util.HashMap; -import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.Map; -import java.util.Set; +import java.util.Map.Entry; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; @@ -37,7 +37,7 @@ public class TopLevelAllocator implements BufferAllocator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopLevelAllocator.class); private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled(); - private final Set<ChildAllocator> children; + private final Map<ChildAllocator, StackTraceElement[]> childrenMap; private final PooledByteBufAllocator innerAllocator = PooledByteBufAllocatorL.DEFAULT; private final Accountor acct; private final boolean errorOnLeak; @@ -55,7 +55,7 @@ public class TopLevelAllocator implements BufferAllocator { private TopLevelAllocator(long maximumAllocation, boolean errorOnLeak){ this.errorOnLeak = errorOnLeak; this.acct = new Accountor(errorOnLeak, null, null, maximumAllocation, 0); - this.children = ENABLE_ACCOUNTING ? new HashSet<ChildAllocator>() : null; + this.childrenMap = ENABLE_ACCOUNTING ? new IdentityHashMap<ChildAllocator, StackTraceElement[]>() : null; } public TopLevelAllocator(DrillConfig config) { @@ -99,16 +99,24 @@ public class TopLevelAllocator implements BufferAllocator { }; logger.debug("New child allocator with initial reservation {}", initialReservation); ChildAllocator allocator = new ChildAllocator(handle, acct, maximumReservation, initialReservation); - if(ENABLE_ACCOUNTING) children.add(allocator); + if(ENABLE_ACCOUNTING) childrenMap.put(allocator, Thread.currentThread().getStackTrace()); + return allocator; } @Override public void close() { if (ENABLE_ACCOUNTING) { - for (ChildAllocator child : children) { - if (!child.isClosed()) { - throw new IllegalStateException("Failure while trying to close allocator: Child level allocators not closed."); + for (Entry<ChildAllocator, StackTraceElement[]> child : childrenMap.entrySet()) { + if (!child.getKey().isClosed()) { + StringBuilder sb = new StringBuilder(); + StackTraceElement[] elements = child.getValue(); + for (int i = 0; i < elements.length; i++) { + sb.append("\t\t"); + sb.append(elements[i]); + sb.append("\n"); + } + throw new IllegalStateException("Failure while trying to close allocator: Child level allocators not closed. Stack trace: \n" + sb); } } }
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index 2b4743d..659a20c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -78,7 +78,6 @@ public class FragmentContext implements Closeable { public FragmentContext(DrillbitContext dbContext, PlanFragment fragment, UserClientConnection connection, FunctionImplementationRegistry funcRegistry) throws OutOfMemoryException, ExecutionSetupException { - this.loader = new QueryClassLoader(true); this.transformer = new ClassTransformer(); this.stats = new FragmentStats(dbContext.getMetrics()); this.context = dbContext; @@ -101,6 +100,7 @@ public class FragmentContext implements Closeable { throw new ExecutionSetupException("Failure while reading plan options.", e); } this.allocator = context.getAllocator().getChildAllocator(fragment.getHandle(), fragment.getMemInitial(), fragment.getMemMax()); + this.loader = new QueryClassLoader(sessionOptions); } public OptionManager getOptions(){ @@ -179,13 +179,7 @@ public class FragmentContext implements Closeable { } public <T> T getImplementationClass(CodeGenerator<T> cg) throws ClassTransformationException, IOException { - long t1 = System.nanoTime(); - - T t = transformer.getImplementationClass(this.loader, cg.getDefinition(), cg.generate(), - cg.getMaterializedClassName()); - logger.debug("Compile time: {} millis.", (System.nanoTime() - t1) / 1000 / 1000); - return t; - + return transformer.getImplementationClass(this.loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName()); } /** http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java index 7ae189e..49d08d9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FragmentOptionsManager.java @@ -51,7 +51,7 @@ public class FragmentOptionsManager implements OptionManager{ @Override public OptionValue getOption(String name) { OptionValue value = options.get(name); - if (value == null) { + if (value == null && systemOptions != null) { value = systemOptions.getOption(name); } return value; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java index 993cead..b5c914e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.server.options; import java.util.Iterator; import java.util.Map; -import java.util.Map.Entry; import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.eigenbase.sql.SqlLiteral; @@ -46,7 +45,7 @@ public class SessionOptionManager implements OptionManager{ @Override public OptionValue getOption(String name) { OptionValue opt = options.get(name); - if(opt == null){ + if(opt == null && systemOptions != null){ return systemOptions.getOption(name); }else{ return opt; @@ -59,7 +58,6 @@ public class SessionOptionManager implements OptionManager{ setValidatedOption(value); } - @Override public OptionList getSessionOptionList() { OptionList list = new OptionList(); @@ -94,5 +92,4 @@ public class SessionOptionManager implements OptionManager{ return systemOptions; } - } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 802f4d9..48f0df0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.cache.DistributedCache.CacheConfig; +import org.apache.drill.exec.compile.QueryClassLoader; import org.apache.drill.exec.planner.fragment.SimpleParallelizer; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.server.options.OptionValue.OptionType; @@ -69,7 +70,9 @@ public class SystemOptionManager implements OptionManager{ ExecConstants.QUEUE_TIMEOUT, ExecConstants.SMALL_QUEUE_SIZE, ExecConstants.MIN_HASH_TABLE_SIZE, - ExecConstants.MAX_HASH_TABLE_SIZE + ExecConstants.MAX_HASH_TABLE_SIZE, + QueryClassLoader.JAVA_COMPILER_VALIDATOR, + QueryClassLoader.JAVA_COMPILER_JANINO_MAXSIZE }; public final PStoreConfig<OptionValue> config; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java index 7d4b657..51421a7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java @@ -122,6 +122,7 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid Thread.currentThread().setName(originalThread); if(!closed) { try { + root.stop(); if(context.isFailed()) { internalFail(context.getFailureCause()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index 07a2075..5acb596 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -64,7 +64,7 @@ public class BaseTestQuery extends ExecTest{ @SuppressWarnings("serial") private static final Properties TEST_CONFIGURATIONS = new Properties() { { - put("drill.exec.sys.store.provider.local.write", "false"); + put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false"); } }; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index dd2c91e..beac02f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -17,32 +17,59 @@ */ package org.apache.drill.exec.compile; +import org.apache.drill.BaseTestQuery; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.compile.sig.GeneratorMapping; import org.apache.drill.exec.compile.sig.MappingSet; -import org.apache.drill.exec.expr.CodeGenerator; import org.apache.drill.exec.expr.ClassGenerator; +import org.apache.drill.exec.expr.CodeGenerator; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.SessionOptionManager; +import org.junit.BeforeClass; import org.junit.Test; -public class TestClassTransformation extends ExecTest{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestClassTransformation.class); +public class TestClassTransformation extends BaseTestQuery { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestClassTransformation.class); + + private static final int ITERATION_COUNT = 1; + + private static SessionOptionManager sessionOptions; + @BeforeClass + public static void beforeTestClassTransformation() throws Exception { + sessionOptions = new SessionOptionManager(bit.getContext().getOptionManager()); + } + @Test + public void testJaninoClassCompiler() throws Exception { + logger.debug("Testing JaninoClassCompiler"); + sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JANINO.name())); + QueryClassLoader loader = new QueryClassLoader(sessionOptions); + for (int i = 0; i < ITERATION_COUNT; i++) { + compilationInnerClass(loader); + } + } + @Test + public void testJDKClassCompiler() throws Exception { + logger.debug("Testing JDKClassCompiler"); + sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JDK.name())); + QueryClassLoader loader = new QueryClassLoader(sessionOptions); + for (int i = 0; i < ITERATION_COUNT; i++) { + compilationInnerClass(loader); + } + } /** * Do a test of a three level class to ensure that nested code generators works correctly. * @throws Exception */ - @Test - public void testInnerClassCompilation() throws Exception{ + public void compilationInnerClass(QueryClassLoader loader) throws Exception{ final TemplateClassDefinition<ExampleInner> template = new TemplateClassDefinition<>(ExampleInner.class, ExampleTemplateWithInner.class); - - ClassTransformer ct = new ClassTransformer(); - QueryClassLoader loader = new QueryClassLoader(true); - CodeGenerator<ExampleInner> cg = CodeGenerator.get(template, new FunctionImplementationRegistry(DrillConfig.create())); + CodeGenerator<ExampleInner> cg = CodeGenerator.get(template, bit.getContext().getFunctionImplementationRegistry()); ClassGenerator<ExampleInner> root = cg.getRoot(); root.setMappingSet(new MappingSet(new GeneratorMapping("doOutside", null, null, null))); @@ -57,9 +84,11 @@ public class TestClassTransformation extends ExecTest{ doubleInner.setMappingSet(new MappingSet(new GeneratorMapping("doDouble", null, null, null))); doubleInner.getSetupBlock().directStatement("System.out.println(\"double\");"); + ClassTransformer ct = new ClassTransformer(); ExampleInner t = ct.getImplementationClass(loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName()); + t.doOutside(); t.doInsideOutside(); - } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02f1c82d/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java new file mode 100644 index 0000000..fecf66b --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.compile; + +import java.util.List; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.proto.UserBitShared.QueryType; +import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.junit.Test; + +public class TestLargeFileCompilation extends BaseTestQuery { + + private static final String LARGE_QUERY; + + private static final int ITERATION_COUNT = 1; + + static { + StringBuilder sb = new StringBuilder("select \n"); + for (int i = 0; i < 300; i++) { + sb.append("\temployee_id+").append(i).append(" as col").append(i).append(",\n"); + } + sb.append("\tfull_name\nfrom cp.`employee.json` limit 1"); + LARGE_QUERY = sb.toString(); + } + + @Test + public void testWithJDK() throws Exception { + test(String.format("alter session set `%s`='JDK'", QueryClassLoader.JAVA_COMPILER_OPTION)); + runTest(); + } + + @Test(expected=org.apache.drill.exec.rpc.RpcException.class) + public void testWithJanino() throws Exception { + test(String.format("alter session set `%s`='JANINO'", QueryClassLoader.JAVA_COMPILER_OPTION)); + runTest(); + } + + private void runTest() throws Exception { + for (int i = 0; i < ITERATION_COUNT; i++) { + List<QueryResultBatch> results = client.runQuery(QueryType.SQL, LARGE_QUERY); + for (QueryResultBatch queryResultBatch : results) { + queryResultBatch.release(); + } + } + } + +}
