[
https://issues.apache.org/jira/browse/HIVE-22977?focusedWorklogId=837799&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-837799
]
ASF GitHub Bot logged work on HIVE-22977:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 09/Jan/23 05:09
Start Date: 09/Jan/23 05:09
Worklog Time Spent: 10m
Work Description: SourabhBadhya commented on code in PR #3801:
URL: https://github.com/apache/hive/pull/3801#discussion_r1064287906
##########
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java:
##########
@@ -2951,4 +2952,170 @@ public void testStatsAfterCompactionPartTbl(boolean
isQueryBased, boolean isAuto
Assert.assertEquals("The number of rows is differing from the expected",
"2", parameters.get("numRows"));
executeStatementOnDriver("drop table if exists " + tblName, driver);
}
+
+ @Test
+ public void testMajorCompactionWithMergeNotPartitionedWithoutBuckets()
throws Exception {
+ testCompactionWithMerge(CompactionType.MAJOR, false, false, null,
Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("base_0000003_v0000007"), true, true);
+ }
+
+ @Test
+ public void testMajorCompactionWithMergePartitionedWithoutBuckets() throws
Exception {
+ testCompactionWithMerge(CompactionType.MAJOR, true, false, "ds=today",
Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("base_0000003_v0000007"), true, true);
+ }
+
+ @Test
+ public void testMajorCompactionWithMergeNotPartitionedWithBuckets() throws
Exception {
+ testCompactionWithMerge(CompactionType.MAJOR, false, true, null,
Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("base_0000003_v0000007"), true, true);
+ }
+
+ @Test
+ public void testMajorCompactionWithMergerPartitionedWithBuckets() throws
Exception {
+ testCompactionWithMerge(CompactionType.MAJOR, true, true, "ds=today",
Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("base_0000003_v0000007"), true, true);
+ }
+
+ @Test
+ public void testMinorCompactionWithMergeNotPartitionedWithoutBuckets()
throws Exception {
+ testCompactionWithMerge(CompactionType.MINOR, false, false, null,
+ Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("delta_0000001_0000003_v0000007"), true,
true);
+ }
+
+ @Test
+ public void testMinorCompactionWithMergePartitionedWithoutBuckets() throws
Exception {
+ testCompactionWithMerge(CompactionType.MINOR, true, false, "ds=today",
+ Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("delta_0000001_0000003_v0000007"), true,
true);
+ }
+
+ @Test
+ public void testMinorCompactionWithMergeNotPartitionedWithBuckets() throws
Exception {
+ testCompactionWithMerge(CompactionType.MINOR, false, true, null,
+ Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("delta_0000001_0000003_v0000007"), true,
true);
+ }
+
+ @Test
+ public void testMinorCompactionWithMergePartitionedWithBuckets() throws
Exception {
+ testCompactionWithMerge(CompactionType.MINOR, true, true, "ds=today",
+ Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("delta_0000001_0000003_v0000007"), true,
true);
+ }
+
+ @Test
+ public void testMajorCompactionAfterMinorWithMerge() throws Exception {
+ testCompactionWithMerge(CompactionType.MINOR, true, true, "ds=today",
+ Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("delta_0000001_0000003_v0000007"),true,
false);
+ testCompactionWithMerge(CompactionType.MAJOR, true, true, "ds=today",
+ Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000003_v0000007",
"delta_0000004_0000004_0000", "delta_0000005_0000005_0000",
+ "delta_0000006_0000006_0000"),
Collections.singletonList("base_0000006_v0000014"), false, true);
+ }
+
+ @Test
+ public void testMinorCompactionAfterMajorWithMerge() throws Exception {
+ testCompactionWithMerge(CompactionType.MAJOR, false, false, null,
+ Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("base_0000003_v0000007"), true, false);
+ testCompactionWithMerge(CompactionType.MINOR, false, false, null,
+ Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000004_0000004_0000",
"delta_0000005_0000005_0000", "delta_0000006_0000006_0000"),
+ Collections.singletonList("delta_0000001_0000006_v0000014"),
false, true);
+ }
+
+ @Test
+ public void testMultipleMajorCompactionWithMerge() throws Exception {
+ testCompactionWithMerge(CompactionType.MAJOR, true, true, "ds=today",
+ Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("base_0000003_v0000007"), true, false);
+ testCompactionWithMerge(CompactionType.MAJOR, true, true, "ds=today",
+ Arrays.asList("bucket_00000", "bucket_00001"),
+ Arrays.asList("delta_0000004_0000004_0000",
"delta_0000005_0000005_0000", "delta_0000006_0000006_0000"),
+ Collections.singletonList("base_0000006_v0000014"), false, true);
+ }
+
+ @Test
+ public void testMultipleMinorCompactionWithMerge() throws Exception {
+ testCompactionWithMerge(CompactionType.MINOR, false, false, null,
+ Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000", "delta_0000003_0000003_0000"),
+ Collections.singletonList("delta_0000001_0000003_v0000007"), true,
false);
+ testCompactionWithMerge(CompactionType.MINOR, false, false, null,
+ Collections.singletonList("bucket_00000"),
+ Arrays.asList("delta_0000001_0000003_v0000007",
"delta_0000004_0000004_0000", "delta_0000005_0000005_0000",
+ "delta_0000006_0000006_0000"),
+ Collections.singletonList("delta_0000001_0000006_v0000014"),
false, true);
+ }
+
+ private void testCompactionWithMerge(CompactionType compactionType, boolean
isPartitioned, boolean isBucketed,
+ String partitionName, List<String>
bucketName, List<String> deltaDirNames, List<String> compactDirNames,
+ boolean createTable, boolean dropTable)
throws Exception {
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_MERGE_COMPACTION_ENABLED, true);
+ String dbName = "default";
+ String tableName = "testCompaction";
+ // Create test table
+ TestDataProvider dataProvider = new TestDataProvider();
+ if (createTable) {
+ dataProvider.createFullAcidTable(tableName, isPartitioned, isBucketed);
+ }
+ // Find the location of the table
+ IMetaStoreClient metaStoreClient = new HiveMetaStoreClient(conf);
+ Table table = metaStoreClient.getTable(dbName, tableName);
+ FileSystem fs = FileSystem.get(conf);
+ // Insert test data into test table
+ dataProvider.insertMmTestData(tableName, isPartitioned);
Review Comment:
Changed it to insertTestData. Done.
Issue Time Tracking
-------------------
Worklog Id: (was: 837799)
Time Spent: 2h 20m (was: 2h 10m)
> Merge delta files instead of running a query in major/minor compaction
> ----------------------------------------------------------------------
>
> Key: HIVE-22977
> URL: https://issues.apache.org/jira/browse/HIVE-22977
> Project: Hive
> Issue Type: Improvement
> Reporter: László Pintér
> Assignee: Sourabh Badhya
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-22977.01.patch, HIVE-22977.02.patch
>
> Time Spent: 2h 20m
> Remaining Estimate: 0h
>
> [Compaction Optimiziation]
> We should analyse the possibility to move a delta file instead of running a
> major/minor compaction query.
> Please consider the following use cases:
> - full acid table but only insert queries were run. This means that no
> delete delta directories were created. Is it possible to merge the delta
> directory contents without running a compaction query?
> - full acid table, initiating queries through the streaming API. If there
> are no abort transactions during the streaming, is it possible to merge the
> delta directory contents without running a compaction query?
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
