[
https://issues.apache.org/jira/browse/HIVE-23365?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17114457#comment-17114457
]
Hive QA commented on HIVE-23365:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/13003729/HIVE-23365.01.patch
{color:green}SUCCESS:{color} +1 due to 1 test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 160 failed/errored test(s), 17282 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[quotedid_basic_standard]
(batchId=4)
org.apache.hadoop.hive.cli.TestHBaseCliDriver.testCliDriver[hbase_bulk]
(batchId=17)
org.apache.hadoop.hive.cli.TestHBaseCliDriver.testCliDriver[hbase_handler_bulk]
(batchId=16)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[reduce_deduplicate_stat_based]
(batchId=23)
org.apache.hadoop.hive.cli.TestNegativeCliDriver.testCliDriver[authorization_delete_nodeletepriv]
(batchId=8)
org.apache.hadoop.hive.cli.TestNegativeCliDriver.testCliDriver[authorization_update_noupdatepriv]
(batchId=8)
org.apache.hadoop.hive.cli.TestNegativeCliDriver.testCliDriver[merge_constraint_notnull]
(batchId=9)
org.apache.hadoop.hive.cli.TestNegativeCliDriver.testCliDriver[subquery_scalar_corr_multi_rows]
(batchId=9)
org.apache.hadoop.hive.cli.TestTezPerfConstraintsCliDriver.testCliDriver[cbo_query24]
(batchId=230)
org.apache.hadoop.hive.cli.TestTezPerfConstraintsCliDriver.testCliDriver[query24]
(batchId=230)
org.apache.hadoop.hive.ql.TestAcidOnTez.testCrudMajorCompactionSplitGrouper
(batchId=179)
org.apache.hadoop.hive.ql.TestTxnCommands.testDelete (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testDeleteIn (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeDeleteUpdate (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeType2SCD01 (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeType2SCD02 (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeUpdateDelete (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeUpdateDeleteNoCardCheck
(batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMoreBucketsThanReducers
(batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testMultipleDelete (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testQuotedIdentifier (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testQuotedIdentifier2 (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testTimeOutReaper (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testUpdateDeleteOfInserts
(batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands.testUpdateOfInserts (batchId=301)
org.apache.hadoop.hive.ql.TestTxnCommands2.testACIDwithSchemaEvolutionAndCompaction
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testBucketCodec (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testCompactWithDelete (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testDeleteIn (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testDynamicPartitionsMerge
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testDynamicPartitionsMerge2
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testEmptyInTblproperties
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMerge (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMerge2 (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMerge3 (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMergeWithPredicate (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMultiInsert (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion02
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion2
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion3
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testOrcNoPPD (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testOrcPPD (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testOriginalFileReaderWhenNonAcidConvertedToAcid
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.testUpdateMixedCase (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.updateDeletePartitioned (batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2.writeBetweenWorkerAndCleaner
(batchId=274)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testACIDwithSchemaEvolutionAndCompaction
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testBucketCodec
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testCompactWithDelete
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testDeleteIn
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testDynamicPartitionsMerge
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testDynamicPartitionsMerge2
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testEmptyInTblproperties
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMerge
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMerge2
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMerge3
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMergeWithPredicate
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMultiInsert
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion02
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion2
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion3
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testOrcNoPPD
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testOrcPPD
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testOriginalFileReaderWhenNonAcidConvertedToAcid
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testUpdateMixedCase
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.updateDeletePartitioned
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.writeBetweenWorkerAndCleaner
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands3.testDeleteEventPruningOff
(batchId=297)
org.apache.hadoop.hive.ql.TestTxnCommands3.testDeleteEventPruningOn
(batchId=297)
org.apache.hadoop.hive.ql.TestTxnCommands3.testRenameTable (batchId=297)
org.apache.hadoop.hive.ql.TestTxnCommands3.testSdpoBucketed (batchId=297)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testDelete
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testDeleteIn
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeDeleteUpdate
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeType2SCD01
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeType2SCD02
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeUpdateDelete
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeUpdateDeleteNoCardCheck
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMoreBucketsThanReducers
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMultipleDelete
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testQuotedIdentifier
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testQuotedIdentifier2
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testTimeOutReaper
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testUpdateDeleteOfInserts
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testUpdateOfInserts
(batchId=277)
org.apache.hadoop.hive.ql.TestTxnConcatenate.testConcatenate (batchId=258)
org.apache.hadoop.hive.ql.TestTxnConcatenate.testConcatenatePart (batchId=258)
org.apache.hadoop.hive.ql.TestTxnExIm.testExportMerge (batchId=278)
org.apache.hadoop.hive.ql.TestTxnExIm.testImport (batchId=278)
org.apache.hadoop.hive.ql.TestTxnExIm.testImportNoTarget (batchId=278)
org.apache.hadoop.hive.ql.TestTxnExIm.testImportVectorized (batchId=278)
org.apache.hadoop.hive.ql.TestTxnLoadData.loadDataUpdate (batchId=248)
org.apache.hadoop.hive.ql.TestTxnLoadData.loadDataUpdateVectorized (batchId=248)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testNoBuckets (batchId=275)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testNoBucketsDP (batchId=275)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testNonAcidToAcidVectorzied
(batchId=275)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testToAcidConversion02 (batchId=275)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testToAcidConversionMultiBucket
(batchId=275)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testNoBuckets (batchId=276)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testNoBucketsDP
(batchId=276)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testNonAcidToAcidVectorzied
(batchId=276)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testToAcidConversion02
(batchId=276)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testToAcidConversionMultiBucket
(batchId=276)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testCheckExpectedLocks
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testCheckExpectedLocksSharedWrite
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testLockConflictDbTable
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testLocksInSubquery
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testLocksInSubquerySharedWrite
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMerge3Way (batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMerge3WayConflict
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMerge3WayConflictSharedWrite
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMergePartitioned
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMergePartitionedConflict
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMergePartitionedConflictSharedWrite
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMergeUnpartitioned
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMergeUnpartitionedConflict
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testMergeUnpartitionedConflictSharedWrite
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testUpdateSelectUpdate
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testUpdateSelectUpdateSharedWrite
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking1
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking10
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking11
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking2
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking3
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking4
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking5
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking6
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking7
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking8
(batchId=302)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2.testWriteSetTracking9
(batchId=302)
org.apache.hadoop.hive.ql.parse.TestMetaStoreEventListenerInRepl.testReplEvents
(batchId=183)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesBootstrapWithJsonMessage.retryIncBootstrapAcidFromDifferentDumpWithoutCleanTablesConfig
(batchId=185)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesBootstrapWithJsonMessage.testAcidTablesBootstrapDuringIncremental
(batchId=185)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesBootstrapWithJsonMessage.testAcidTablesBootstrapDuringIncrementalWithOpenTxnsTimeout
(batchId=185)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesBootstrapWithJsonMessage.testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites
(batchId=185)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesBootstrapWithJsonMessage.testRetryAcidTablesBootstrapFromDifferentDump
(batchId=185)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesWithJsonMessage.testAcidTablesBootstrap
(batchId=202)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTables.testAcidTablesBootstrap
(batchId=204)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTablesBootstrap.retryIncBootstrapAcidFromDifferentDumpWithoutCleanTablesConfig
(batchId=182)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTablesBootstrap.testAcidTablesBootstrapDuringIncremental
(batchId=182)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTablesBootstrap.testAcidTablesBootstrapDuringIncrementalWithOpenTxnsTimeout
(batchId=182)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTablesBootstrap.testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites
(batchId=182)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTablesBootstrap.testRetryAcidTablesBootstrapFromDifferentDump
(batchId=182)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosIncrementalLoadAcidTables.testAcidTableIncrementalReplication
(batchId=205)
org.apache.hadoop.hive.ql.parse.TestStatsReplicationScenariosACID.testForParallelBootstrapLoad
(batchId=187)
org.apache.hadoop.hive.ql.parse.TestStatsReplicationScenariosACID.testMetadataOnlyDump
(batchId=187)
org.apache.hadoop.hive.ql.parse.TestStatsReplicationScenariosACID.testNonParallelBootstrapLoad
(batchId=187)
org.apache.hadoop.hive.ql.parse.TestStatsReplicationScenariosACID.testRetryFailure
(batchId=187)
org.apache.hadoop.hive.ql.security.authorization.plugin.TestHiveAuthorizerCheckInvocation.testDelete
(batchId=210)
org.apache.hadoop.hive.ql.security.authorization.plugin.TestHiveAuthorizerCheckInvocation.testUpdateSomeColumnsUsed
(batchId=210)
org.apache.hadoop.hive.ql.security.authorization.plugin.TestHiveAuthorizerCheckInvocation.testUpdateSomeColumnsUsedExprInSet
(batchId=210)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningDelete
(batchId=180)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningUpdate
(batchId=180)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.schemaEvolutionAddColDynamicPartitioningUpdate
(batchId=180)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testCompactionDataLoadedWithInsertOverwrite
(batchId=180)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testMinorCompactionForSplitUpdateWithInsertsAndDeletes
(batchId=180)
org.apache.hive.hcatalog.streaming.TestStreaming.testNoBuckets (batchId=152)
org.apache.hive.jdbc.TestJdbcDriver2.testExceucteUpdateCounts (batchId=219)
org.apache.hive.streaming.TestStreaming.testAutoRollTransactionBatch
(batchId=311)
org.apache.hive.streaming.TestStreaming.testNoBuckets (batchId=311)
{noformat}
Test results:
https://builds.apache.org/job/PreCommit-HIVE-Build/22555/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/22555/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-22555/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.YetusPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 160 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 13003729 - PreCommit-HIVE-Build
> Put RS deduplication optimization under cost based decision
> -----------------------------------------------------------
>
> Key: HIVE-23365
> URL: https://issues.apache.org/jira/browse/HIVE-23365
> Project: Hive
> Issue Type: Improvement
> Components: Physical Optimizer
> Reporter: Jesus Camacho Rodriguez
> Assignee: Stamatis Zampetakis
> Priority: Major
> Attachments: HIVE-23365.01.patch
>
>
> Currently, RS deduplication is always executed whenever it is semantically
> correct. However, it could be beneficial to leave both RS operators in the
> plan, e.g., if the NDV of the second RS is very low. Thus, we would like this
> decision to be cost-based. We could use a simple heuristic that would work
> fine for most of the cases without introducing regressions for existing
> cases, e.g., if NDV for partition column is less than estimated parallelism
> in the second RS, do not execute deduplication.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
