[
https://issues.apache.org/jira/browse/HIVE-23052?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17064769#comment-17064769
]
Hive QA commented on HIVE-23052:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12997404/HIVE-23052.3.patch
{color:red}ERROR:{color} -1 due to no test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 77 failed/errored test(s), 18125 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[autoColumnStats_4]
(batchId=14)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[constant_prop_timestamp_date_cast]
(batchId=55)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[explain_locks]
(batchId=51)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[mm_all] (batchId=78)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[stats_nonpart]
(batchId=14)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[stats_sizebug]
(batchId=94)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[mm_all]
(batchId=164)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[acid_vectorization_original]
(batchId=190)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[check_constraint]
(batchId=171)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[default_constraint]
(batchId=177)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[dynamic_semijoin_reduction_3]
(batchId=189)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[enforce_constraint_notnull]
(batchId=171)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[insert_into_default_keyword]
(batchId=166)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[insert_only_empty_query]
(batchId=189)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[materialized_view_create_rewrite_4]
(batchId=170)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[materialized_view_create_rewrite_5]
(batchId=167)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[materialized_view_rewrite_window]
(batchId=189)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[runtime_stats_merge]
(batchId=192)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[schq_materialized]
(batchId=184)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[semijoin_hint]
(batchId=174)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[sqlmerge]
(batchId=191)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[sqlmerge_stats]
(batchId=190)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_acid4]
(batchId=168)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_mapjoin_complex_values]
(batchId=185)
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[acid_vectorization_original_tez]
(batchId=117)
org.apache.hadoop.hive.metastore.TestHiveMetaStoreTxns.testLocksWithTxn
(batchId=241)
org.apache.hadoop.hive.metastore.txn.TestCompactionTxnHandler.testFindPotentialCompactions
(batchId=317)
org.apache.hadoop.hive.metastore.txn.TestCompactionTxnHandler.testMarkCleanedCleansTxnsAndTxnComponents
(batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testCheckLockAcquireAfterWaiting
(batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testCheckLockTxnAborted
(batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testLockEESW (batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testLockESRSW (batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testLockSRSW (batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testLockSWSR (batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testLockSWSWSR (batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testLockSWSWSW (batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testUnlockOnAbort
(batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testUnlockOnCommit
(batchId=317)
org.apache.hadoop.hive.metastore.txn.TestTxnHandler.testUnlockWithTxn
(batchId=317)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeOnTezEdges (batchId=362)
org.apache.hadoop.hive.ql.TestTxnCommands2.testOrcPPD (batchId=343)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testOrcPPD
(batchId=357)
org.apache.hadoop.hive.ql.TestTxnCommandsForMmTable.testInsertOverwriteForPartitionedMmTable
(batchId=317)
org.apache.hadoop.hive.ql.TestTxnCommandsForOrcMmTable.testInsertOverwriteForPartitionedMmTable
(batchId=339)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeOnTezEdges
(batchId=343)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testNoBuckets (batchId=343)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testToAcidConversionMultiBucket
(batchId=343)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testNoBuckets (batchId=343)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testToAcidConversionMultiBucket
(batchId=343)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testDelete (batchId=362)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testReadWrite (batchId=362)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testRollback (batchId=362)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testSingleWritePartition
(batchId=362)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testSingleWriteTable
(batchId=362)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testUpdate (batchId=362)
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager.testWriteDynamicPartition
(batchId=362)
org.apache.hadoop.hive.ql.parse.TestScheduledReplicationScenarios.testAcidTablesReplLoadBootstrapIncr
(batchId=270)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.chooseMajorOverMinorWhenBothValid
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.cleanEmptyAbortedTxns
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.compactPartitionHighDeltaPct
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.compactPartitionTooManyDeltas
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.compactTableHighDeltaPct
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.compactTableTooManyDeltas
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.dropPartition
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.dropTable (batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.enoughDeltasNoBase
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.majorCompactOnPartitionTooManyAborts
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.majorCompactOnTableTooManyAborts
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactOnManyDifferentPartitionAborts
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactTableDeltaPctNotHighEnough
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactTableDynamicPartitioning
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactTableNotEnoughDeltas
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactWhenCompactAlreadyScheduled
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactWhenNoCompactSet
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.noCompactWhenNoCompactSetLowerCase
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.processCompactionCandidatesInParallel
(batchId=328)
org.apache.hadoop.hive.ql.txn.compactor.TestInitiator.twoTxnsOnSamePartitionGenerateOneCompactionRequest
(batchId=328)
{noformat}
Test results:
https://builds.apache.org/job/PreCommit-HIVE-Build/21231/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/21231/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-21231/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.YetusPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 77 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12997404 - PreCommit-HIVE-Build
> Optimize lock enqueueing in TxnHandler
> --------------------------------------
>
> Key: HIVE-23052
> URL: https://issues.apache.org/jira/browse/HIVE-23052
> Project: Hive
> Issue Type: Improvement
> Reporter: Marton Bod
> Assignee: Marton Bod
> Priority: Major
> Attachments: HIVE-23052.1.patch, HIVE-23052.2.patch,
> HIVE-23052.3.patch
>
>
> * Reduce scope of next_lock_id select-for-update by moving the txn_component
> inserts before the S4U + inserting the hive_locks entries before the S4U
> (first with a temp ID, which will be replaced later in a single update). This
> helps decrease the overall time that the next_lock_id table is locked,
> thereby increasing concurrency
> * Insert txn_components in a batch instead of one-by-one
> * Increment next_lock_id and update hive_locks table in a single batch
> statement
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
