[
https://issues.apache.org/jira/browse/HIVE-16757?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16025243#comment-16025243
]
Hive QA commented on HIVE-16757:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12869905/HIVE-16757.02.patch
{color:red}ERROR:{color} -1 due to no test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 135 failed/errored test(s), 10763 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[cbo_const] (batchId=17)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[cbo_input26] (batchId=2)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[cbo_rp_auto_join1]
(batchId=3)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[constprog3] (batchId=33)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[cross_join_merge]
(batchId=6)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[dynamic_rdd_cache]
(batchId=51)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[filter_cond_pushdown]
(batchId=57)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby_join_pushdown]
(batchId=75)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[infer_bucket_sort]
(batchId=78)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[infer_join_preds]
(batchId=23)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[join26] (batchId=18)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[join45] (batchId=19)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[join_cond_pushdown_1]
(batchId=34)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[join_cond_pushdown_3]
(batchId=23)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[mergejoin] (batchId=56)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[mergejoins] (batchId=2)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[optimize_filter_literal]
(batchId=51)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[order_by_expr_1]
(batchId=65)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[pcr] (batchId=57)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_join2] (batchId=40)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_join3] (batchId=18)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_outer_join4]
(batchId=63)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_udf_case]
(batchId=41)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_coalesce]
(batchId=10)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_date_1]
(batchId=20)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_interval_arithmetic]
(batchId=4)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[dynamic_semijoin_user_level]
(batchId=142)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[explainuser_2]
(batchId=142)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_join]
(batchId=153)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_limit]
(batchId=161)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_rp_join]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_rp_limit]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_rp_semijoin]
(batchId=145)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_rp_views]
(batchId=147)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_semijoin]
(batchId=151)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_views]
(batchId=150)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[constprog_semijoin]
(batchId=156)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[dynamic_semijoin_reduction]
(batchId=153)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[explainuser_1]
(batchId=151)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[limit_join_transpose]
(batchId=148)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[lineage3]
(batchId=153)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[mergejoin]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[semijoin_hint]
(batchId=149)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[subquery_scalar]
(batchId=152)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_coalesce]
(batchId=145)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_date_1]
(batchId=147)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_interval_arithmetic]
(batchId=144)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_join30]
(batchId=150)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[constprog_semijoin]
(batchId=168)
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[explainanalyze_2]
(batchId=99)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query13]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query14]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query15]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query16]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query17]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query18]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query19]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query1] (batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query20]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query21]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query22]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query23]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query24]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query25]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query26]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query27]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query29]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query30]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query31]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query32]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query33]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query34]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query36]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query37]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query38]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query39]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query3] (batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query40]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query42]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query43]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query46]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query48]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query50]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query51]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query52]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query54]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query55]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query56]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query58]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query5] (batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query60]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query64]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query65]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query66]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query67]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query68]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query69]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query6] (batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query70]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query71]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query72]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query73]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query75]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query76]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query79]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query7] (batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query80]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query81]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query82]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query84]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query85]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query86]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query87]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query88]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query89]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query8] (batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query90]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query91]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query93]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query94]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query95]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query96]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query97]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query98]
(batchId=232)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[cbo_limit]
(batchId=138)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[cbo_semijoin]
(batchId=116)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[dynamic_rdd_cache]
(batchId=123)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join26]
(batchId=108)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join_cond_pushdown_1]
(batchId=115)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join_cond_pushdown_3]
(batchId=111)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[mergejoins]
(batchId=100)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[pcr] (batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[ppd_join2]
(batchId=118)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[ppd_join3]
(batchId=108)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[ppd_outer_join4]
(batchId=128)
{noformat}
Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/5431/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/5431/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-5431/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 135 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12869905 - PreCommit-HIVE-Build
> Use memoization in HiveRelMdRowCount.getRowCount
> ------------------------------------------------
>
> Key: HIVE-16757
> URL: https://issues.apache.org/jira/browse/HIVE-16757
> Project: Hive
> Issue Type: Bug
> Components: Query Planning
> Reporter: Remus Rusanu
> Assignee: Remus Rusanu
> Attachments: HIVE-16757.01.patch, HIVE-16757.02.patch
>
>
> On complex queries HiveRelMdRowCount.getRowCount can get called many times.
> since it does not memoize its result and the call is recursive, it results in
> an explosion of calls. for example a query with 49 joins, during join
> ordering (LoptOtimizerJoinRule) the HiveRelMdRowCount.getRowCount gets called
> 6442 as a top level call, but the recursivity exploded this to 501729 calls.
> Memoization of the rezult would stop the recursion early. In my testing this
> reduced the join reordering time for said query from 11s to <1s..
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
