[
https://issues.apache.org/jira/browse/HIVE-18009?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16244956#comment-16244956
]
Aihua Xu commented on HIVE-18009:
---------------------------------
Thanks [~xuefuz] I will update if I need to update anything.
> Multiple lateral view query is slow on hive on spark
> ----------------------------------------------------
>
> Key: HIVE-18009
> URL: https://issues.apache.org/jira/browse/HIVE-18009
> Project: Hive
> Issue Type: Improvement
> Components: Spark
> Affects Versions: 3.0.0
> Reporter: Aihua Xu
> Assignee: Aihua Xu
> Attachments: HIVE-18009.1.patch, HIVE-18009.2.patch
>
>
> When running the query with multiple lateral view, HoS is busy with the
> compilation. GenSparkUtils has an efficient implementation of
> getChildOperator when we have diamond hierarchy in operator trees (lateral
> view in this case) since the node may be visited multiple times.
> {noformat}
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:442)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWork.process(GenSparkWork.java:104)
> at
> org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
> at
> org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:90)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
> at
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.startWalking(GenSparkWorkWalker.java:72)
> at
> org.apache.hadoop.hive.ql.parse.spark.SparkCompiler.generateTaskTree(SparkCompiler.java:225)
> at
> org.apache.hadoop.hive.ql.parse.TaskCompiler.compile(TaskCompiler.java:204)
> at
> org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10083)
> at
> org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:9876)
> at
> org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:222)
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:445)
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:311)
> at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1189)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1284)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1115)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1103)
> at
> org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:220)
> at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:172)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:383)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:318)
> at
> org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:416)
> at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:432)
> at
> org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:726)
> at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:693)
> at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:628)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
