Rajesh Balamohan created HIVE-22879:
---------------------------------------
Summary: Optimise jar file loading in CalcitePlanner
Key: HIVE-22879
URL: https://issues.apache.org/jira/browse/HIVE-22879
Project: Hive
Issue Type: Improvement
Components: CBO
Reporter: Rajesh Balamohan
{{CalcitePlanner }} internally uses {{org.codehaus.janino.UnitCompiler (calcite
dependency)}} and this appears to load the jars in every thread. Need to check
if this can be avoided.
Here is an example.
{noformat}
at java.util.zip.ZipFile.getEntry(Native Method)
at java.util.zip.ZipFile.getEntry(ZipFile.java:310)
- locked <0x00000005c1af21c0> (a java.util.jar.JarFile)
at java.util.jar.JarFile.getEntry(JarFile.java:240)
at java.util.jar.JarFile.getJarEntry(JarFile.java:223)
at sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:1005)
at sun.misc.URLClassPath.getResource(URLClassPath.java:212)
at java.net.URLClassLoader$1.run(URLClassLoader.java:365)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- locked <0x00000005caa3be88> (a java.lang.Object)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at
org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89)
at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:312)
- locked <0x0000000686136868> (a
org.codehaus.janino.ClassLoaderIClassLoader)
at
org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8556)
at
org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8478)
at
org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8471)
at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8331)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6855)
at org.codehaus.janino.UnitCompiler.access$14200(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6497)
at
org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6494)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4224)
at
org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6494)
at
org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6490)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:4148)
at
org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at
org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9026)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7106)
at org.codehaus.janino.UnitCompiler.access$15800(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6517)
at
org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6490)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
at
org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at
org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
at
org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9237)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9123)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9025)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5062)
at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4423)
at
org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4396)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5622)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5592)
at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4434)
at
org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4396)
at org.codehaus.janino.Java$NewInitializedArray.accept(Java.java:5373)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2649)
at org.codehaus.janino.UnitCompiler.access$2800(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1504)
at
org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1487)
at org.codehaus.janino.Java$ReturnStatement.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
at
org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
at
org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:465)
at
org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313)
at
org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235)
at org.apache.calcite.rex.RexExecutable.compile(RexExecutable.java:60)
at org.apache.calcite.rex.RexExecutable.<init>(RexExecutable.java:49)
at
org.apache.calcite.rex.RexExecutorImpl.reduce(RexExecutorImpl.java:127)
at
org.apache.calcite.rex.RexSimplify.simplifyCast(RexSimplify.java:1810)
at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:270)
at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
at
org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:321)
at
org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:313)
at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:288)
at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
at org.apache.calcite.rex.RexSimplify.simplifyAnds(RexSimplify.java:463)
at org.apache.calcite.rex.RexUtil.simplifyAnds(RexUtil.java:1748)
at
org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:759)
at
org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:749)
at
org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule.onMatch(HiveSubQueryRemoveRule.java:115)
at
org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(AbstractRelOptPlanner.java:319)
at org.apache.calcite.plan.hep.HepPlanner.applyRule(HepPlanner.java:560)
at
org.apache.calcite.plan.hep.HepPlanner.applyRules(HepPlanner.java:419)
at
org.apache.calcite.plan.hep.HepPlanner.executeInstruction(HepPlanner.java:256)
at
org.apache.calcite.plan.hep.HepInstruction$RuleInstance.execute(HepInstruction.java:127)
at
org.apache.calcite.plan.hep.HepPlanner.executeProgram(HepPlanner.java:215)
at
org.apache.calcite.plan.hep.HepPlanner.findBestExp(HepPlanner.java:202)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2518)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2484)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.hepPlan(CalcitePlanner.java:2459)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1814)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1731)
at
org.apache.calcite.tools.Frameworks.lambda$withPlanner$0(Frameworks.java:130)
at
org.apache.calcite.tools.Frameworks$$Lambda$162/385313421.apply(Unknown Source)
at
org.apache.calcite.prepare.CalcitePrepareImpl.perform(CalcitePrepareImpl.java:915)
at org.apache.calcite.tools.Frameworks.withPrepare(Frameworks.java:179)
at org.apache.calcite.tools.Frameworks.withPlanner(Frameworks.java:125)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner.logicalPlan(CalcitePlanner.java:1492)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:468)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12488)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:358)
at
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
at
org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:171)
at
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:219)
at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:103)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:215)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:828)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:774)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:768)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:125)
at
org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:203)
at
org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:325)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1688)
at
org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:345)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
