This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e213351b6c HIVE-26023: Non blocking REPLACE, RENAME COLUMNS (Denys
Kuzmenko, reviewed by Antal Sinkovits, Laszlo Vegh)
e213351b6c is described below
commit e213351b6cf0a7d202eb21f15ef39eaa7e5cf168
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Thu May 5 20:20:16 2022 +0200
HIVE-26023: Non blocking REPLACE, RENAME COLUMNS (Denys Kuzmenko, reviewed
by Antal Sinkovits, Laszlo Vegh)
Closes #3089
---
.../apache/hadoop/hive/ql/hooks/WriteEntity.java | 6 +-
.../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 91 +++++++++++++++++++++-
2 files changed, 93 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
index 8d9079eeb0..567da91b48 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
@@ -206,14 +206,12 @@ public class WriteEntity extends Entity implements
Serializable {
*/
public static WriteType determineAlterTableWriteType(AlterTableType op,
Table table, HiveConf conf) {
switch (op) {
- case RENAME_COLUMN:
case CLUSTERED_BY:
case NOT_SORTED:
case NOT_CLUSTERED:
case SET_FILE_FORMAT:
case SET_SERDE:
case DROPPROPS:
- case REPLACE_COLUMNS:
case ARCHIVE:
case UNARCHIVE:
case ALTERLOCATION:
@@ -228,7 +226,9 @@ public class WriteEntity extends Entity implements
Serializable {
case OWNER:
return WriteType.DDL_EXCLUSIVE;
- case ADDCOLS:
+ case ADDCOLS:
+ case REPLACE_COLUMNS:
+ case RENAME_COLUMN:
case ADD_CONSTRAINT:
case DROP_CONSTRAINT:
case RENAME:
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 58d0edb107..eaf1a3cd73 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -4140,4 +4140,93 @@ public class TestDbTxnManager2 extends
DbTxnManagerEndToEndTestBase{
driver.getFetchTask().fetch(res);
Assert.assertEquals("[1\t2\tNULL, 3\t4\tNULL, 5\t6\t7]", res.toString());
}
-}
+
+ @Test
+ public void testReplaceColumnsNonBlocking() throws Exception {
+ testReplaceColumns(false);
+ }
+ @Test
+ public void testReplaceColumnsBlocking() throws Exception {
+ testReplaceColumns(true);
+ }
+ private void testReplaceColumns(boolean blocking) throws Exception {
+ testReplaceRenameColumns(blocking, "replace columns (c string, a bigint)");
+ }
+
+ @Test
+ public void testRenameColumnsNonBlocking() throws Exception {
+ testRenameColumns(false);
+ }
+ @Test
+ public void testRenameColumnsBlocking() throws Exception {
+ testRenameColumns(true);
+ }
+ private void testRenameColumns(boolean blocking) throws Exception {
+ testReplaceRenameColumns(blocking, "change column a c string");
+ }
+
+ private void testReplaceRenameColumns(boolean blocking, String
alterSubQuery) throws Exception {
+ dropTable(new String[] {"tab_acid"});
+
+ HiveConf.setBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED, !blocking);
+ driver = Mockito.spy(driver);
+
+ HiveConf.setBoolVar(driver2.getConf(),
HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED, !blocking);
+ driver2 = Mockito.spy(driver2);
+
+ driver.run("create table if not exists tab_acid (a int, b int) " +
+ "stored as orc TBLPROPERTIES ('transactional'='true')");
+ driver.run("insert into tab_acid (a,b) values(1,2),(3,4)");
+
+ driver.compileAndRespond("select * from tab_acid");
+ List<String> res = new ArrayList<>();
+
+ driver.lockAndRespond();
+ List<ShowLocksResponseElement> locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+
+ checkLock(LockType.SHARED_READ,
+ LockState.ACQUIRED, "default", "tab_acid", null, locks);
+
+ DbTxnManager txnMgr2 = (DbTxnManager)
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ swapTxnManager(txnMgr2);
+ driver2.compileAndRespond("alter table tab_acid "+ alterSubQuery);
+
+ if (blocking) {
+ txnMgr2.acquireLocks(driver2.getPlan(), ctx, null, false);
+ locks = getLocks();
+
+ ShowLocksResponseElement checkLock = checkLock(LockType.EXCLUSIVE,
+ LockState.WAITING, "default", "tab_acid", null, locks);
+
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+
+ driver.getFetchTask().fetch(res);
+ swapTxnManager(txnMgr2);
+
+ FieldSetter.setField(txnMgr2,
txnMgr2.getClass().getDeclaredField("numStatements"), 0);
+ txnMgr2.getMS().unlock(checkLock.getLockid());
+ }
+ driver2.lockAndRespond();
+ locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", blocking ? 1 : 2,
locks.size());
+
+ checkLock(blocking ? LockType.EXCLUSIVE : LockType.EXCL_WRITE,
+ LockState.ACQUIRED, "default", "tab_acid", null, locks);
+
+ Mockito.doNothing().when(driver2).lockAndRespond();
+ driver2.run();
+
+ if (!blocking) {
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+ }
+ Mockito.reset(driver, driver2);
+
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("[1\t2, 3\t4]", res.toString());
+ }
+}
\ No newline at end of file
