[
https://issues.apache.org/jira/browse/HIVE-19267?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16481758#comment-16481758
]
Hive QA commented on HIVE-19267:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12924257/HIVE-19267.05.patch
{color:green}SUCCESS:{color} +1 due to 4 test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 110 failed/errored test(s), 14380 tests
executed
*Failed tests:*
{noformat}
TestReplicationScenariosAcidTables - did not produce a TEST-*.xml file (likely
timed out) (batchId=232)
org.apache.hadoop.hive.cli.TestAccumuloCliDriver.testCliDriver[accumulo_queries]
(batchId=246)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[autoColumnStats_7]
(batchId=66)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[autoColumnStats_9]
(batchId=38)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[auto_join2] (batchId=66)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[bucketizedhiveinputformat_auto]
(batchId=64)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[constprog2] (batchId=13)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[innerjoin] (batchId=35)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[insert_orig_table]
(batchId=64)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[insert_overwrite_directory2]
(batchId=69)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[leftsemijoin]
(batchId=47)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[merge4] (batchId=12)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[orc_merge_incompat2]
(batchId=86)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parallel] (batchId=26)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parallel_colstats]
(batchId=35)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parallel_join1]
(batchId=5)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_ppd_boolean]
(batchId=61)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_ppd_timestamp]
(batchId=58)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_vectorization_0]
(batchId=17)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[sample4] (batchId=17)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[sample5] (batchId=58)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[skewjoin] (batchId=24)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[skewjoin_onesideskew]
(batchId=73)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[skewjoin_union_remove_1]
(batchId=89)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[skewjoin_union_remove_2]
(batchId=29)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[smb_mapjoin9]
(batchId=42)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[temp_table_gb1]
(batchId=34)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[union_remove_6_subq]
(batchId=40)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_groupby4]
(batchId=16)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_groupby6]
(batchId=90)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_17]
(batchId=90)
org.apache.hadoop.hive.cli.TestHBaseCliDriver.testCliDriver[hbase_bulk]
(batchId=105)
org.apache.hadoop.hive.cli.TestHBaseCliDriver.testCliDriver[hbase_queries]
(batchId=102)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[insert_overwrite_directory2]
(batchId=185)
org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[spark_explainuser_1]
(batchId=184)
org.apache.hadoop.hive.cli.TestMinimrCliDriver.testCliDriver[scriptfile1]
(batchId=95)
org.apache.hadoop.hive.cli.TestNegativeCliDriver.testCliDriver[delete_non_acid_table]
(batchId=98)
org.apache.hadoop.hive.cli.TestNegativeCliDriver.testCliDriver[update_non_acid_table]
(batchId=97)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[runtime_skewjoin_mapjoin_spark]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[skewjoin]
(batchId=119)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[spark_union_merge]
(batchId=131)
org.apache.hadoop.hive.llap.daemon.impl.comparator.TestAMReporter.testMultipleAM
(batchId=310)
org.apache.hadoop.hive.ql.TestAcidOnTez.testAcidInsertWithRemoveUnion
(batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testBucketedAcidInsertWithRemoveUnion
(batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testGetSplitsLocks (batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testInsertWithRemoveUnion (batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testMapJoinOnMR (batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testMapJoinOnTez (batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testMergeJoinOnMR (batchId=229)
org.apache.hadoop.hive.ql.TestAcidOnTez.testMergeJoinOnTez (batchId=229)
org.apache.hadoop.hive.ql.TestTxnCommands.testDeleteIn (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeCase (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeDeleteUpdate (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeType2SCD01 (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeType2SCD02 (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeUpdateDelete (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testMergeUpdateDeleteNoCardCheck
(batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testQuotedIdentifier (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands.testQuotedIdentifier2 (batchId=302)
org.apache.hadoop.hive.ql.TestTxnCommands2.testDeleteIn (batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testDynamicPartitionsMerge
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testDynamicPartitionsMerge2
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testInsertOverwriteWithSelfJoin
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMerge (batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMerge2 (batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMerge3 (batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMergeWithPredicate (batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMultiInsert (batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2.testMultiInsertStatement
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testDeleteIn
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testDynamicPartitionsMerge
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testDynamicPartitionsMerge2
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testInsertOverwriteWithSelfJoin
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMerge
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMerge2
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMerge3
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMergeWithPredicate
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMultiInsert
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testMultiInsertStatement
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testDeleteIn
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeCase
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeDeleteUpdate
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeType2SCD01
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeType2SCD02
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeUpdateDelete
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMergeUpdateDeleteNoCardCheck
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testQuotedIdentifier
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testQuotedIdentifier2
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnLoadData.testValidations (batchId=265)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testInsertFromUnion (batchId=287)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testToAcidConversionMultiBucket
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testInsertFromUnion
(batchId=287)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testToAcidConversionMultiBucket
(batchId=287)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningDelete[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningDelete[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningInsert[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningInsert[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningUpdate[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.dynamicPartitioningUpdate[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.mmTableBucketed[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.mmTableBucketed[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.schemaEvolutionAddColDynamicPartitioningInsert[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.schemaEvolutionAddColDynamicPartitioningInsert[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.schemaEvolutionAddColDynamicPartitioningUpdate[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.schemaEvolutionAddColDynamicPartitioningUpdate[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testStatsAfterCompactionPartTbl[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testStatsAfterCompactionPartTbl[1]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testTableProperties[0]
(batchId=231)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testTableProperties[1]
(batchId=231)
org.apache.hive.service.server.TestInformationSchemaWithPrivilege.test
(batchId=239)
{noformat}
Test results:
https://builds.apache.org/job/PreCommit-HIVE-Build/11080/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/11080/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-11080/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.YetusPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 110 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12924257 - PreCommit-HIVE-Build
> Create/Replicate ACID Write event
> ---------------------------------
>
> Key: HIVE-19267
> URL: https://issues.apache.org/jira/browse/HIVE-19267
> Project: Hive
> Issue Type: Sub-task
> Components: repl, Transactions
> Affects Versions: 3.0.0
> Reporter: mahesh kumar behera
> Assignee: mahesh kumar behera
> Priority: Major
> Labels: ACID, DR, pull-request-available, replication
> Attachments: HIVE-19267.01.patch, HIVE-19267.02.patch,
> HIVE-19267.03.patch, HIVE-19267.04.patch, HIVE-19267.05.patch
>
>
>
> h1. Replicate ACID write Events
> * Create new EVENT_WRITE event with related message format to log the write
> operations with in a txn along with data associated.
> * Log this event when perform any writes (insert into, insert overwrite,
> load table, delete, update, merge, truncate) on table/partition.
> * If a single MERGE/UPDATE/INSERT/DELETE statement operates on multiple
> partitions, then need to log one event per partition.
> * DbNotificationListener should log this type of event to special metastore
> table named "MTxnWriteNotificationLog".
> * This table should maintain a map of txn ID against list of
> tables/partitions written by given txn.
> * The entry for a given txn should be removed by the cleaner thread that
> removes the expired events from EventNotificationTable.
> h1. Replicate Commit Txn operation (with writes)
> Add new EVENT_COMMIT_TXN to log the metadata/data of all tables/partitions
> modified within the txn.
> *Source warehouse:*
> * This event should read the EVENT_WRITEs from "MTxnWriteNotificationLog"
> metastore table to consolidate the list of tables/partitions modified within
> this txn scope.
> * Based on the list of tables/partitions modified and table Write ID, need
> to compute the list of delta files added by this txn.
> * Repl dump should read this message and dump the metadata and delta files
> list.
> *Target warehouse:*
> * Ensure snapshot isolation at target for on-going read txns which shouldn't
> view the data replicated from committed txn. (Ensured with open and allocate
> write ID events).
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
