This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e3751ab5453 HIVE-26374: Query based compaction fails for tables with
CDT and columns with Reserved Keywords (Chiran Ravani, reviewed by Denys
Kuzmenko, Sai Hemanth Gantasala)
e3751ab5453 is described below
commit e3751ab545370f9b252d0b4a07bc315037541a95
Author: Chiran Ravani <[email protected]>
AuthorDate: Fri Jul 15 06:59:37 2022 -0400
HIVE-26374: Query based compaction fails for tables with CDT and columns
with Reserved Keywords (Chiran Ravani, reviewed by Denys Kuzmenko, Sai Hemanth
Gantasala)
Closes #3419
---
.../ql/txn/compactor/TestCrudCompactorOnTez.java | 65 ++++++++++++++++++++++
.../apache/hadoop/hive/ql/exec/DDLPlanUtils.java | 2 +-
.../ql/txn/compactor/CompactionQueryBuilder.java | 15 +++--
3 files changed, 73 insertions(+), 9 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index ece4d5290e1..cffb58bc41c 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -2351,4 +2351,69 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
Assert.assertEquals("base_10000000_v0000009", baseName);
}
+ @Test
+ public void testCompactionShouldNotFailOnStructField() throws Exception {
+ conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true);
+ String dbName = "default";
+ String tblName = "compaction_hive_26374";
+
+ TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ TestDataProvider testDP = new TestDataProvider();
+
+ // Create test table
+ executeStatementOnDriver("drop table if exists " + tblName, driver);
+ executeStatementOnDriver("CREATE TABLE " + tblName + "(col1
array<struct<arr_col1:int, `timestamp`:string>>)" +
+ "STORED AS ORC TBLPROPERTIES('transactional'='true')", driver);
+
+ // Insert test data into test table
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName +
+ " SELECT ARRAY(NAMED_STRUCT('arr_col1',1,'timestamp','2022-07-05
21:51:20.371'))",driver);
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName +
+ " SELECT ARRAY(NAMED_STRUCT('arr_col1',2,'timestamp','2022-07-05
21:51:20.371'))",driver);
+
+ // Find the location of the table
+ IMetaStoreClient msClient = new HiveMetaStoreClient(conf);
+ Table table = msClient.getTable(dbName, tblName);
+ FileSystem fs = FileSystem.get(conf);
+ // Verify deltas (delta_0000001_0000001_0000, delta_0000002_0000002_0000)
are present
+ Assert.assertEquals("Delta directories does not match before compaction",
+ Arrays.asList("delta_0000001_0000001_0000",
"delta_0000002_0000002_0000"),
+ CompactorTestUtil.getBaseOrDeltaNames(fs,
AcidUtils.deltaFileFilter, table, null));
+
+ // Get all data before compaction is run
+ List<String> expectedData = testDP.getAllData(tblName);
+
+ //Do a compaction directly and wait for it to finish
+ CompactionRequest rqst = new CompactionRequest(dbName, tblName,
CompactionType.MAJOR);
+ CompactionResponse resp = txnHandler.compact(rqst);
+ runWorker(conf);
+
+ CompactorTestUtil.runCleaner(conf);
+
+ //Check if the compaction succeed
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<ShowCompactResponseElement> compacts = rsp.getCompacts();
+ Assert.assertEquals("Expecting 1 rows and found " + compacts.size(), 1,
compacts.size());
+ Assert.assertEquals("Expecting compaction state 'succeeded' and found:" +
compacts.get(0).getState(),
+ "succeeded", compacts.get(0).getState());
+ // Should contain only one base directory now
+ FileStatus[] status = fs.listStatus(new Path(table.getSd().getLocation()));
+ int inputFileCount = 0;
+ for(FileStatus file: status) {
+ inputFileCount++;
+ }
+ Assert.assertEquals("Expecting 1 file and found "+ inputFileCount, 1,
inputFileCount);
+
+ // Check bucket file name
+ List<String> baseDir = CompactorTestUtil.getBaseOrDeltaNames(fs,
AcidUtils.baseFileFilter, table, null);
+ List<String> expectedBucketFiles = Arrays.asList("bucket_00000");
+ Assert.assertEquals("Bucket names are not matching after compaction",
expectedBucketFiles,
+ CompactorTestUtil
+ .getBucketFileNames(fs, table, null, baseDir.get(0)));
+
+ // Verify all contents
+ List<String> actualData = testDP.getAllData(tblName);
+ Assert.assertEquals(expectedData, actualData);
+ }
+
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
index 80561179242..62248ff4f71 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
@@ -837,7 +837,7 @@ public class DDLPlanUtils {
/**
* Struct fields are identifiers, need to be put between ``.
*/
- private String formatType(TypeInfo typeInfo) {
+ public static String formatType(TypeInfo typeInfo) {
switch (typeInfo.getCategory()) {
case PRIMITIVE:
return typeInfo.getTypeName();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
index d9aeeeed8d6..8a297533888 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.util.DirectionUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.common.util.HiveStringUtils;
import org.slf4j.Logger;
@@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -418,16 +420,13 @@ class CompactionQueryBuilder {
+ "`currentTransaction` bigint, `row` struct<");
}
List<FieldSchema> cols = sourceTab.getSd().getCols();
- boolean isFirst = true;
+ List<String> columnDescs = new ArrayList<>();
for (FieldSchema col : cols) {
- if (!isFirst) {
- query.append(", ");
- }
- isFirst = false;
- query.append("`").append(col.getName()).append("` ");
- query.append(crud ? ":" : "");
- query.append(col.getType());
+ String columnType =
DDLPlanUtils.formatType(TypeInfoUtils.getTypeInfoFromTypeString(col.getType()));
+ String columnDesc = "`" + col.getName() + "` " + (crud ? ":" : "") +
columnType;
+ columnDescs.add(columnDesc);
}
+ query.append(StringUtils.join(',',columnDescs));
query.append(crud ? ">" : "");
query.append(") ");
}
