[ 
https://issues.apache.org/jira/browse/HIVE-18009?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Aihua Xu updated HIVE-18009:
----------------------------
    Status: Patch Available  (was: In Progress)

> Multiple lateral view query is slow on hive on spark
> ----------------------------------------------------
>
>                 Key: HIVE-18009
>                 URL: https://issues.apache.org/jira/browse/HIVE-18009
>             Project: Hive
>          Issue Type: Improvement
>          Components: Spark
>    Affects Versions: 3.0.0
>            Reporter: Aihua Xu
>            Assignee: Aihua Xu
>         Attachments: HIVE-18009.1.patch, HIVE-18009.2.patch, 
> HIVE-18009.3.patch
>
>
> When running the query with multiple lateral view, HoS is busy with the 
> compilation. GenSparkUtils has an efficient implementation of 
> getChildOperator when we have diamond hierarchy in operator trees (lateral 
> view in this case) since the node may be visited multiple times.
> {noformat}
> at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:442)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWork.process(GenSparkWork.java:104)
>       at 
> org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
>       at 
> org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:90)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.startWalking(GenSparkWorkWalker.java:72)
>       at 
> org.apache.hadoop.hive.ql.parse.spark.SparkCompiler.generateTaskTree(SparkCompiler.java:225)
>       at 
> org.apache.hadoop.hive.ql.parse.TaskCompiler.compile(TaskCompiler.java:204)
>       at 
> org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10083)
>       at 
> org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:9876)
>       at 
> org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:222)
>       at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:445)
>       at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:311)
>       at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1189)
>       at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1284)
>       at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1115)
>       at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1103)
>       at 
> org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:220)
>       at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:172)
>       at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:383)
>       at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:318)
>       at 
> org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:416)
>       at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:432)
>       at 
> org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:726)
>       at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:693)
>       at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:628)
>       at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>       at 
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>       at 
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>       at java.lang.reflect.Method.invoke(Method.java:498)
>       at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
>       at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Reply via email to