[
https://issues.apache.org/jira/browse/HIVE-17035?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16080221#comment-16080221
]
Hive QA commented on HIVE-17035:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12876397/HIVE-17035.4.patch
{color:red}ERROR:{color} -1 due to no test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 74 failed/errored test(s), 10834 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[join]
(batchId=240)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[auto_sortmerge_join_2]
(batchId=46)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[extract] (batchId=3)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[llap_smb]
(batchId=143)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[dynamic_semijoin_reduction_3]
(batchId=158)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[dynpart_sort_optimization2]
(batchId=144)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[reduce_deduplicate_extended]
(batchId=156)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[schema_evol_orc_acid_table_update]
(batchId=157)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[schema_evol_orc_acidvec_table_update]
(batchId=144)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[tez_join_result_complex]
(batchId=146)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[tez_union_multiinsert]
(batchId=151)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[update_all_partitioned]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[infer_bucket_sort_map_operators]
(batchId=168)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[infer_bucket_sort_reducers_power_two]
(batchId=167)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[vector_outer_join5]
(batchId=169)
org.apache.hadoop.hive.cli.TestMinimrCliDriver.testCliDriver[infer_bucket_sort_map_operators]
(batchId=86)
org.apache.hadoop.hive.cli.TestMinimrCliDriver.testCliDriver[infer_bucket_sort_reducers_power_two]
(batchId=86)
org.apache.hadoop.hive.cli.TestNegativeMinimrCliDriver.testCliDriver[mapreduce_stack_trace]
(batchId=91)
org.apache.hadoop.hive.cli.TestNegativeMinimrCliDriver.testCliDriver[mapreduce_stack_trace_turnoff]
(batchId=91)
org.apache.hadoop.hive.cli.TestNegativeMinimrCliDriver.testCliDriver[minimr_broken_pipe]
(batchId=91)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query14]
(batchId=232)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query23]
(batchId=232)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[add_part_multiple]
(batchId=129)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[auto_join26]
(batchId=106)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[auto_join8]
(batchId=136)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[auto_smb_mapjoin_14]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[bucketmapjoin5]
(batchId=136)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[date_udf]
(batchId=114)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[filter_join_breaktask2]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby4_map]
(batchId=120)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby4_map_skew]
(batchId=124)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_cube1]
(batchId=101)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_multi_single_reducer]
(batchId=126)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_rollup1]
(batchId=114)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[input_part2]
(batchId=120)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join27]
(batchId=117)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join30]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join32]
(batchId=108)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join32_lessSize]
(batchId=103)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join33]
(batchId=107)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join38]
(batchId=134)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join8] (batchId=120)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[join_map_ppr]
(batchId=132)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_gby2]
(batchId=116)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_with_join]
(batchId=128)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[pcr] (batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[ptf] (batchId=107)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[sample1]
(batchId=105)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[skewjoin_union_remove_1]
(batchId=137)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[smb_mapjoin_11]
(batchId=100)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[smb_mapjoin_12]
(batchId=104)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[smb_mapjoin_7]
(batchId=124)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[stats1]
(batchId=106)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[timestamp_1]
(batchId=112)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[timestamp_2]
(batchId=100)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[timestamp_3]
(batchId=115)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[timestamp_lazy]
(batchId=123)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[timestamp_udf]
(batchId=134)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union10]
(batchId=105)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union12]
(batchId=102)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union17]
(batchId=130)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union18]
(batchId=107)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union19]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union22]
(batchId=106)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union31]
(batchId=101)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union4]
(batchId=131)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union6]
(batchId=113)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_23]
(batchId=131)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_short_regress]
(batchId=121)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorized_ptf]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[windowing]
(batchId=122)
org.apache.hive.hcatalog.api.TestHCatClient.testPartitionRegistrationWithCustomSchema
(batchId=177)
org.apache.hive.hcatalog.api.TestHCatClient.testPartitionSpecRegistrationWithCustomSchema
(batchId=177)
org.apache.hive.hcatalog.api.TestHCatClient.testTableSchemaPropagation
(batchId=177)
{noformat}
Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/5930/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/5930/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-5930/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 74 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12876397 - PreCommit-HIVE-Build
> Optimizer: Lineage transform() should be invoked after rest of the optimizers
> are invoked
> -----------------------------------------------------------------------------------------
>
> Key: HIVE-17035
> URL: https://issues.apache.org/jira/browse/HIVE-17035
> Project: Hive
> Issue Type: Bug
> Components: Logical Optimizer
> Reporter: Rajesh Balamohan
> Assignee: Rajesh Balamohan
> Priority: Minor
> Attachments: HIVE-17035.1.patch, HIVE-17035.2.patch,
> HIVE-17035.3.patch, HIVE-17035.4.patch
>
>
> In a fairly large query which had tens of left join, time taken to create
> linageInfo itself took 1500+ seconds. This is due to the fact that the table
> had lots of columns and in some processing, it ended up processing 7000+
> value columns in {{ReduceSinkLineage}}, though only 50 columns were projected
> in the query.
> It would be good to invoke lineage transform when rest of the optimizers in
> {{Optimizer}} are invoked. This would avoid unwanted processing and help in
> improving the runtime.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
