[
https://issues.apache.org/jira/browse/HIVE-17963?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16238007#comment-16238007
]
Hive QA commented on HIVE-17963:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12895494/HIVE-17963.1.patch
{color:green}SUCCESS:{color} +1 due to 2 test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 101 failed/errored test(s), 11353 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[insert_values_orig_table_use_metadata]
(batchId=62)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[tez_union_dynamic_partition]
(batchId=146)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[tez_union_dynamic_partition_2]
(batchId=146)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[autoColumnStats_1]
(batchId=153)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[autoColumnStats_2]
(batchId=166)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[dp_counter_non_mm]
(batchId=153)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[ppd_union_view]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[sysdb]
(batchId=156)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[tez_dml]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[spark_dynamic_partition_pruning_2]
(batchId=175)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[vector_outer_join1]
(batchId=174)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[vector_outer_join2]
(batchId=173)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[vector_outer_join3]
(batchId=174)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[vector_outer_join4]
(batchId=176)
org.apache.hadoop.hive.cli.TestNegativeMinimrCliDriver.testCliDriver[ct_noperm_loc]
(batchId=94)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[cbo_union]
(batchId=137)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[column_access_stats]
(batchId=128)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_sort_1_23]
(batchId=138)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_sort_skew_1_23]
(batchId=107)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[load_dyn_part13]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[load_dyn_part14]
(batchId=142)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert]
(batchId=118)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_move_tasks_share_dependencies]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[optimize_nullscan]
(batchId=138)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[semijoin]
(batchId=113)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[skewjoin_union_remove_1]
(batchId=141)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[spark_combine_equivalent_work_2]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[spark_union_merge]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[stats1]
(batchId=109)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[subquery_multi]
(batchId=111)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[table_access_keys_stats]
(batchId=135)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union10]
(batchId=108)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union12]
(batchId=105)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union13]
(batchId=126)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union18]
(batchId=110)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union19]
(batchId=129)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union22]
(batchId=109)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union24]
(batchId=129)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union28]
(batchId=123)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union29]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union30]
(batchId=138)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union32]
(batchId=116)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union33]
(batchId=114)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union34]
(batchId=108)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union4]
(batchId=136)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union6]
(batchId=117)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union8]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union] (batchId=104)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_date]
(batchId=123)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_date_trim]
(batchId=138)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_null]
(batchId=139)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_10]
(batchId=114)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_11]
(batchId=112)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_12]
(batchId=122)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_13]
(batchId=141)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_14]
(batchId=108)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_15]
(batchId=141)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_16]
(batchId=135)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_17]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_18]
(batchId=106)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_19]
(batchId=112)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_1]
(batchId=117)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_20]
(batchId=140)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_21]
(batchId=130)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_22]
(batchId=138)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_23]
(batchId=136)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_24]
(batchId=109)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_25]
(batchId=142)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_2]
(batchId=120)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_3]
(batchId=126)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_4]
(batchId=107)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_5]
(batchId=129)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_6]
(batchId=122)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_6_subq]
(batchId=119)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_7]
(batchId=104)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_8]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_remove_9]
(batchId=135)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_script]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_top_level]
(batchId=130)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[union_view]
(batchId=109)
org.apache.hadoop.hive.cli.control.TestDanglingQOuts.checkDanglingQOut
(batchId=206)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testDelete (batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testDeleteDynamicPartitioning
(batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testInsert (batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testInsertDynamicPartitioning
(batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testNonAcidDynamicPartitioning
(batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testNonAcidWrite
(batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testUpdate (batchId=276)
org.apache.hadoop.hive.ql.exec.TestFileSinkOperator.testUpdateDynamicPartitioning
(batchId=276)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testAmPoolInteractions
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testApplyPlanQpChanges
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testApplyPlanUserMapping
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testAsyncSessionInitFailures
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testClusterFractions
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testDestroyAndReturn
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testQueueing
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testReopen (batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testReuse (batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testReuseWithDifferentPool
(batchId=281)
org.apache.hadoop.hive.ql.exec.tez.TestWorkloadManager.testReuseWithQueueing
(batchId=281)
org.apache.hadoop.hive.ql.parse.TestReplicationScenarios.testConstraints
(batchId=223)
{noformat}
Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/7614/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/7614/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-7614/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 101 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12895494 - PreCommit-HIVE-Build
> Fix for HIVE-17113 can be improved for non-blobstore filesystems
> ----------------------------------------------------------------
>
> Key: HIVE-17963
> URL: https://issues.apache.org/jira/browse/HIVE-17963
> Project: Hive
> Issue Type: Bug
> Reporter: Jason Dere
> Assignee: Jason Dere
> Priority: Major
> Attachments: HIVE-17963.1.patch
>
>
> HIVE-17113/HIVE-17813 fix the duplicate file issue by performing file moves
> on a file-by-file basis. For non-blobstore filesystems this results in many
> more filesystem/namenode operations compared to the previous
> Utilities.mvFileToFinalPath() behavior (dedup files in src dir, rename src
> dir to final dir).
> For non-blobstore filesystems, a better solution would be the one described
> [here|https://issues.apache.org/jira/browse/HIVE-17113?focusedCommentId=16100564&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16100564]:
> 1) Move the temp directory to a new directory name, to prevent additional
> files from being added by any runaway processes.
> 2) Run removeTempOrDuplicateFiles() on this renamed temp directory
> 3) Run renameOrMoveFiles() to move the renamed temp directory to the final
> location.
> This results in only one additional file operation in non-blobstore FSes
> compared to the original Utilities.mvFileToFinalPath() behavior.
> The proposal is to do away with the config setting
> hive.exec.move.files.from.source.dir and always have behavior that should
> take care of the duplicate file issue described in HIVE-17113. For
> non-blobstore filesystems we will do steps 1-3 described above. For blobstore
> filesystems we will do the solution done in HIVE-17113/HIVE-17813 which does
> the file-by-file copy - this should have the same number of file operations
> as doing a rename directory on blobstore, which effectively results in file
> moves on a file-by-file basis.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
