This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new eed1f99ac71 HIVE-26882: Allow transactional check of Table parameter
before altering the Table (#3888) (Peter Vary reviewed by Prasanth Jayachandran
and Szehon Ho) (#3944)
eed1f99ac71 is described below
commit eed1f99ac71d89b01c67f81c3a002996c44dddc1
Author: pvary <[email protected]>
AuthorDate: Sun Jan 15 09:14:04 2023 +0100
HIVE-26882: Allow transactional check of Table parameter before altering
the Table (#3888) (Peter Vary reviewed by Prasanth Jayachandran and Szehon Ho)
(#3944)
---
.../thrift/gen-cpp/hive_metastore_constants.cpp | 4 +
.../gen/thrift/gen-cpp/hive_metastore_constants.h | 2 +
.../metastore/api/hive_metastoreConstants.java | 4 +
.../src/gen/thrift/gen-php/metastore/Types.php | 10 ++
.../gen/thrift/gen-py/hive_metastore/constants.py | 2 +
.../gen/thrift/gen-rb/hive_metastore_constants.rb | 4 +
.../hadoop/hive/metastore/HiveAlterHandler.java | 19 +++-
.../apache/hadoop/hive/metastore/ObjectStore.java | 28 +++++-
.../org/apache/hadoop/hive/metastore/RawStore.java | 16 +++-
.../src/main/thrift/hive_metastore.thrift | 3 +
.../client/TestTablesCreateDropAlterTruncate.java | 102 +++++++++++++++++++++
11 files changed, 186 insertions(+), 8 deletions(-)
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
index 1c1b3ce5eeb..875e272eb4a 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
@@ -61,6 +61,10 @@ hive_metastoreConstants::hive_metastoreConstants() {
TABLE_BUCKETING_VERSION = "bucketing_version";
+ EXPECTED_PARAMETER_KEY = "expected_parameter_key";
+
+ EXPECTED_PARAMETER_VALUE = "expected_parameter_value";
+
}
}}} // namespace
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
index 1f062530e4d..4cffc9491fb 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
@@ -40,6 +40,8 @@ class hive_metastoreConstants {
std::string TABLE_NO_AUTO_COMPACT;
std::string TABLE_TRANSACTIONAL_PROPERTIES;
std::string TABLE_BUCKETING_VERSION;
+ std::string EXPECTED_PARAMETER_KEY;
+ std::string EXPECTED_PARAMETER_VALUE;
};
extern const hive_metastoreConstants g_hive_metastore_constants;
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
index 2ee81df1dc1..76be19a7d0e 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
@@ -86,4 +86,8 @@ import org.slf4j.LoggerFactory;
public static final String TABLE_BUCKETING_VERSION = "bucketing_version";
+ public static final String EXPECTED_PARAMETER_KEY = "expected_parameter_key";
+
+ public static final String EXPECTED_PARAMETER_VALUE =
"expected_parameter_value";
+
}
diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index 84f7e3320c8..8c65e346a27 100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@ -31377,6 +31377,8 @@ final class Constant extends \Thrift\Type\TConstant {
static protected $TABLE_NO_AUTO_COMPACT;
static protected $TABLE_TRANSACTIONAL_PROPERTIES;
static protected $TABLE_BUCKETING_VERSION;
+ static protected $EXPECTED_PARAMETER_KEY;
+ static protected $EXPECTED_PARAMETER_VALUE;
static protected function init_DDL_TIME() {
return "transient_lastDdlTime";
@@ -31477,6 +31479,14 @@ final class Constant extends \Thrift\Type\TConstant {
static protected function init_TABLE_BUCKETING_VERSION() {
return "bucketing_version";
}
+
+ static protected function init_EXPECTED_PARAMETER_KEY() {
+ return "expected_parameter_key";
+ }
+
+ static protected function init_EXPECTED_PARAMETER_VALUE() {
+ return "expected_parameter_value";
+ }
}
diff --git
a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
index c27745a6078..261b3bf82aa 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
@@ -34,3 +34,5 @@ TABLE_IS_TRANSACTIONAL = "transactional"
TABLE_NO_AUTO_COMPACT = "no_auto_compaction"
TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties"
TABLE_BUCKETING_VERSION = "bucketing_version"
+EXPECTED_PARAMETER_KEY = "expected_parameter_key"
+EXPECTED_PARAMETER_VALUE = "expected_parameter_value"
diff --git
a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
index 9e6cedd43d7..8cc22cfcace 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
@@ -57,3 +57,7 @@ TABLE_TRANSACTIONAL_PROPERTIES = %q"transactional_properties"
TABLE_BUCKETING_VERSION = %q"bucketing_version"
+EXPECTED_PARAMETER_KEY = %q"expected_parameter_key"
+
+EXPECTED_PARAMETER_VALUE = %q"expected_parameter_value"
+
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index f328ad18155..f010f0d2cca 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import javax.jdo.Constants;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -146,7 +147,17 @@ public class HiveAlterHandler implements AlterHandler {
rename = true;
}
- msdb.openTransaction();
+ String expectedKey = environmentContext != null &&
environmentContext.getProperties() != null ?
+
environmentContext.getProperties().get(hive_metastoreConstants.EXPECTED_PARAMETER_KEY)
: null;
+ String expectedValue = environmentContext != null &&
environmentContext.getProperties() != null ?
+
environmentContext.getProperties().get(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE)
: null;
+
+ if (expectedKey != null) {
+ // If we have to check the expected state of the table we have to
prevent nonrepeatable reads.
+ msdb.openTransaction(Constants.TX_REPEATABLE_READ);
+ } else {
+ msdb.openTransaction();
+ }
// get old table
oldt = msdb.getTable(catName, dbname, name);
if (oldt == null) {
@@ -154,6 +165,12 @@ public class HiveAlterHandler implements AlterHandler {
Warehouse.getCatalogQualifiedTableName(catName, dbname, name) + "
doesn't exist");
}
+ if (expectedKey != null && expectedValue != null
+ && !expectedValue.equals(oldt.getParameters().get(expectedKey)))
{
+ throw new MetaException("The table has been modified. The parameter
value for key '" + expectedKey + "' is '"
+ + oldt.getParameters().get(expectedKey) + "'. The expected was
value was '" + expectedValue + "'");
+ }
+
if (oldt.getPartitionKeysSize() != 0) {
isPartitionedTable = true;
}
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 99fc74e6d45..815ee4184d8 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -704,17 +704,31 @@ public class ObjectStore implements RawStore,
Configurable {
}
/**
- * Opens a new one or the one already created Every call of this function
must
- * have corresponding commit or rollback function call
+ * Opens a new one or the one already created. Every call of this function
must
+ * have corresponding commit or rollback function call.
*
* @return an active transaction
*/
-
@Override
public boolean openTransaction() {
+ return openTransaction(null);
+ }
+
+ /**
+ * Opens a new one or the one already created. Every call of this function
must
+ * have corresponding commit or rollback function call.
+ *
+ * @param isolationLevel The transaction isolation level. Only possible to
set on the first call.
+ * @return an active transaction
+ */
+ @Override
+ public boolean openTransaction(String isolationLevel) {
openTrasactionCalls++;
if (openTrasactionCalls == 1) {
currentTransaction = pm.currentTransaction();
+ if (isolationLevel != null) {
+ currentTransaction.setIsolationLevel(isolationLevel);
+ }
currentTransaction.begin();
transactionStatus = TXN_STATUS.OPEN;
} else {
@@ -724,10 +738,16 @@ public class ObjectStore implements RawStore,
Configurable {
throw new RuntimeException("openTransaction called in an interior"
+ " transaction scope, but currentTransaction is not active.");
}
+
+ // Can not change the isolation level on an already open transaction
+ if (isolationLevel != null &&
!isolationLevel.equals(currentTransaction.getIsolationLevel())) {
+ throw new RuntimeException("Can not set isolation level on an open
transaction");
+ }
}
boolean result = currentTransaction.isActive();
- debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive
= " + result);
+ debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive
= " + result + ", isolationLevel = "
+ + currentTransaction.getIsolationLevel());
return result;
}
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index f350aa9fd7c..a6ffcf417e9 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -104,14 +104,24 @@ public interface RawStore extends Configurable {
void shutdown();
/**
- * Opens a new one or the one already created Every call of this function
must
- * have corresponding commit or rollback function call
+ * Opens a new one or the one already created. Every call of this function
must
+ * have corresponding commit or rollback function call.
*
* @return an active transaction
*/
-
boolean openTransaction();
+ /**
+ * Opens a new one or the one already created. Every call of this function
must
+ * have corresponding commit or rollback function call.
+ *
+ * @param isolationLevel The transaction isolation level. Only possible to
set on the first call.
+ * @return an active transaction
+ */
+ default boolean openTransaction(String isolationLevel) {
+ throw new UnsupportedOperationException("Setting isolation level for this
Store is not supported");
+ }
+
/**
* if this is the commit of the first open call then an actual commit is
* called.
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift
b/standalone-metastore/src/main/thrift/hive_metastore.thrift
index ad1dc1f7696..c08a03c0e1a 100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@ -2243,3 +2243,6 @@ const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",
const string TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties",
const string TABLE_BUCKETING_VERSION = "bucketing_version",
+// Keys for alter table environment context parameters
+const string EXPECTED_PARAMETER_KEY = "expected_parameter_key",
+const string EXPECTED_PARAMETER_VALUE = "expected_parameter_value",
\ No newline at end of file
diff --git
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
index c1eff55b11c..a2ab9e807f3 100644
---
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
+++
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
@@ -23,7 +23,9 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.ColumnType;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.RawStore;
import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Catalog;
@@ -33,6 +35,7 @@ import
org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -65,15 +68,21 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+import static org.junit.Assert.assertTrue;
/**
* Test class for IMetaStoreClient API. Testing the Table related functions
for metadata
@@ -1075,6 +1084,99 @@ public class TestTablesCreateDropAlterTruncate extends
MetaStoreClientTest {
}
}
+ @Test
+ public void testAlterTableExpectedPropertyMatch() throws Exception {
+ Table originalTable = testTables[0];
+
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY,
"transient_lastDdlTime");
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE,
+ originalTable.getParameters().get("transient_lastDdlTime"));
+
+ client.alter_table(originalTable.getCatName(), originalTable.getDbName(),
originalTable.getTableName(),
+ originalTable, context);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testAlterTableExpectedPropertyDifferent() throws Exception {
+ Table originalTable = testTables[0];
+
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY,
"transient_lastDdlTime");
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE,
"alma");
+
+ client.alter_table(originalTable.getCatName(), originalTable.getDbName(),
originalTable.getTableName(),
+ originalTable, context);
+ }
+
+ /**
+ * This tests ensures that concurrent Iceberg commits will fail. Acceptable
as a first sanity check.
+ * <p>
+ * I have not found a good way to check that HMS side database commits are
parallel in the
+ * {@link
org.apache.hadoop.hive.metastore.HiveAlterHandler#alterTable(RawStore,
Warehouse, String, String, String, Table, EnvironmentContext)}
+ * call, but this test could be used to manually ensure that using
breakpoints.
+ */
+ @Test
+ public void testAlterTableExpectedPropertyConcurrent() throws Exception {
+ Table originalTable = testTables[0];
+
+ originalTable.getParameters().put("snapshot", "0");
+ client.alter_table(originalTable.getCatName(), originalTable.getDbName(),
originalTable.getTableName(),
+ originalTable, null);
+
+ ExecutorService threads = null;
+ try {
+ threads = Executors.newFixedThreadPool(2);
+ for (int i = 0; i < 3; i++) {
+ EnvironmentContext context = new EnvironmentContext();
+
context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY,
"snapshot");
+
context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE,
String.valueOf(i));
+
+ Table newTable = originalTable.deepCopy();
+ newTable.getParameters().put("snapshot", String.valueOf(i + 1));
+
+ IMetaStoreClient client1 = metaStore.getClient();
+ IMetaStoreClient client2 = metaStore.getClient();
+
+ Collection<Callable<Boolean>> concurrentTasks = new ArrayList<>(2);
+ concurrentTasks.add(alterTask(client1, newTable, context));
+ concurrentTasks.add(alterTask(client2, newTable, context));
+
+ Collection<Future<Boolean>> results =
threads.invokeAll(concurrentTasks);
+
+ boolean foundSuccess = false;
+ boolean foundFailure = false;
+
+ for (Future<Boolean> result : results) {
+ if (result.get()) {
+ foundSuccess = true;
+ } else {
+ foundFailure = true;
+ }
+ }
+
+ assertTrue("At least one success is expected", foundSuccess);
+ assertTrue("At least one failure is expected", foundFailure);
+ }
+ } finally {
+ if (threads != null) {
+ threads.shutdown();
+ }
+ }
+ }
+
+ private Callable<Boolean> alterTask(IMetaStoreClient hmsClient, Table
newTable, EnvironmentContext context) {
+ return () -> {
+ try {
+ hmsClient.alter_table(newTable.getCatName(), newTable.getDbName(),
newTable.getTableName(),
+ newTable, context);
+ } catch (Throwable e) {
+ return false;
+ }
+ return true;
+ };
+ }
+
@Test
public void tablesInOtherCatalogs() throws TException, URISyntaxException {
String catName = "create_etc_tables_in_other_catalogs";
