This is an automated email from the ASF dual-hosted git repository.
mbod pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f0a98c7 HIVE-25989: HBase CTLT failure rollback should not drop the
underlying HBase table (#3076) (Marton Bod, reviewed by Peter Vary)
f0a98c7 is described below
commit f0a98c7f03153982adee4b8f5fb698b4775ab924
Author: Marton Bod <[email protected]>
AuthorDate: Thu Mar 10 12:09:54 2022 +0100
HIVE-25989: HBase CTLT failure rollback should not drop the underlying
HBase table (#3076) (Marton Bod, reviewed by Peter Vary)
---
hbase-handler/pom.xml | 6 +
.../apache/hadoop/hive/hbase/HBaseMetaHook.java | 12 +-
.../apache/hadoop/hive/hbase/TestHBaseQueries.java | 132 +++++++++++++++++++++
.../ddl/table/create/like/CreateTableLikeDesc.java | 2 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 1 +
.../thrift/gen-cpp/hive_metastore_constants.cpp | 2 +
.../gen/thrift/gen-cpp/hive_metastore_constants.h | 1 +
.../metastore/api/hive_metastoreConstants.java | 2 +
.../src/gen/thrift/gen-php/metastore/Constant.php | 6 +
.../gen/thrift/gen-py/hive_metastore/constants.py | 1 +
.../gen/thrift/gen-rb/hive_metastore_constants.rb | 2 +
.../src/main/thrift/hive_metastore.thrift | 1 +
.../apache/hadoop/hive/metastore/HMSHandler.java | 2 +
13 files changed, 169 insertions(+), 1 deletion(-)
diff --git a/hbase-handler/pom.xml b/hbase-handler/pom.xml
index d01b3b9..bd14c62 100644
--- a/hbase-handler/pom.xml
+++ b/hbase-handler/pom.xml
@@ -246,6 +246,12 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-standalone-metastore-server</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>${basedir}/src/java</sourceDirectory>
diff --git
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseMetaHook.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseMetaHook.java
index f01ed57..4e9f507 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseMetaHook.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseMetaHook.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.hbase;
+import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -41,6 +42,8 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTLT;
+
/**
* MetaHook for HBase. Updates the table data in HBase too. Not thread safe,
and cleanup should
* be used after usage.
@@ -49,6 +52,7 @@ public class HBaseMetaHook implements HiveMetaHook, Closeable
{
private static final Logger LOG =
LoggerFactory.getLogger(HBaseMetaHook.class);
private Configuration hbaseConf;
private Admin admin;
+ private boolean isCTLT;
public HBaseMetaHook(Configuration hbaseConf) {
this.hbaseConf = hbaseConf;
@@ -124,6 +128,12 @@ public class HBaseMetaHook implements HiveMetaHook,
Closeable {
if (tbl.getSd().getLocation() != null) {
throw new MetaException("LOCATION may not be specified for HBase.");
}
+ // we shouldn't delete the hbase table in case of CREATE TABLE ... LIKE
(CTLT) failures since we'd remove the
+ // hbase table for the original table too. Let's save the value of CTLT
here, because this param will be gone
+ isCTLT = Optional.ofNullable(tbl.getParameters())
+ .map(params -> params.get(TABLE_IS_CTLT))
+ .map(Boolean::parseBoolean)
+ .orElse(false);
org.apache.hadoop.hbase.client.Table htable = null;
@@ -187,7 +197,7 @@ public class HBaseMetaHook implements HiveMetaHook,
Closeable {
String tableName = getHBaseTableName(table);
boolean isPurge = !MetaStoreUtils.isExternalTable(table) ||
MetaStoreUtils.isExternalTablePurge(table);
try {
- if (isPurge &&
getHBaseAdmin().tableExists(TableName.valueOf(tableName))) {
+ if (isPurge && getHBaseAdmin().tableExists(TableName.valueOf(tableName))
&& !isCTLT) {
// we have created an HBase table, so we delete it to roll back;
if (getHBaseAdmin().isTableEnabled(TableName.valueOf(tableName))) {
getHBaseAdmin().disableTable(TableName.valueOf(tableName));
diff --git
a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java
b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java
new file mode 100644
index 0000000..da69f08
--- /dev/null
+++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseQueries.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.DriverFactory;
+import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
+import
org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestHBaseQueries {
+
+ private static MiniZooKeeperCluster zooKeeperCluster;
+ private static MiniHBaseCluster miniHBaseCluster;
+ private final HiveConf baseConf;
+ private IDriver driver;
+
+ /**
+ * Test class for running queries using HBase tables. Creates a mini ZK and
an HBase test cluster.
+ * Each test method must instantiate a driver object first, using either the
baseConf or a modified version of that.
+ * Once finished, each test method must take care of cleaning up any
database objects they've created (tables,
+ * databases, etc.), otherwise those will be visible for subsequent test
methods too.
+ */
+ public TestHBaseQueries() throws Exception {
+ baseConf = new HiveConf(HBaseConfiguration.create(),
TestHBaseQueries.class);
+ baseConf.set(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER.varname,
SQLStdHiveAuthorizerFactory.class.getName());
+
+ // set up Zookeeper
+ File tmpDir = Files.createTempDirectory(Paths.get("/tmp/"),
"tmp_").toFile();
+ System.setProperty("zookeeper.4lw.commands.whitelist", "stat");
+ zooKeeperCluster = new MiniZooKeeperCluster();
+ int zkPort = zooKeeperCluster.startup(tmpDir);
+ baseConf.setInt("hbase.zookeeper.property.clientPort", zkPort);
+
+ // set up HBase
+ baseConf.setBoolean("hbase.netty.nativetransport", false);
+ HBaseTestingUtility util = new HBaseTestingUtility(baseConf);
+ util.setZkCluster(zooKeeperCluster);
+ miniHBaseCluster = util.startMiniHBaseCluster(1, 1);
+
+ // set up HMS backend DB
+ TestTxnDbUtil.setConfValues(baseConf);
+ TestTxnDbUtil.cleanDb(baseConf);
+ TestTxnDbUtil.prepDb(baseConf);
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ if (miniHBaseCluster != null) {
+ miniHBaseCluster.shutdown();
+ }
+ if (zooKeeperCluster != null) {
+ zooKeeperCluster.shutdown();
+ }
+ }
+
+ @Before
+ public void before() {
+ SessionState.start(baseConf);
+ }
+
+ @After
+ public void after() throws Exception {
+ if (driver != null) {
+ driver.close();
+ }
+ TestTxnDbUtil.cleanDb(baseConf);
+ }
+
+ @Test
+ public void testRollbackDoesNotDeleteOriginTableWhenCTLTFails() throws
CommandProcessorException {
+ HiveConf conf = new HiveConf(baseConf);
+ conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER,
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES, true);
+ conf.setBoolVar(HiveConf.ConfVars.CREATE_TABLES_AS_ACID, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY, true);
+ conf.setVar(HiveConf.ConfVars.HIVEDEFAULTMANAGEDFILEFORMAT, "ORC");
+
+ driver = DriverFactory.newDriver(conf);
+
+ driver.run("CREATE EXTERNAL TABLE `old_hbase_hive_table`(\n" +
+ " `key` int COMMENT '',\n" +
+ " `value` string COMMENT '')\n" +
+ " ROW FORMAT SERDE\n" +
+ " 'org.apache.hadoop.hive.hbase.HBaseSerDe'\n" +
+ " STORED BY\n" +
+ " 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'\n" +
+ " WITH SERDEPROPERTIES (\n" +
+ " 'hbase.columns.mapping'=':key,cf:cf')\n" +
+ " TBLPROPERTIES ('hbase.table.name'='hbase_hive_table')");
+ try {
+ driver.run("create table new_hbase_hive_table like
old_hbase_hive_table");
+ } catch (Exception e) {
+ // expected - above command will try to create a transactional table but
will fail
+ }
+ // original table is still present, rollback() has not deleted it
+ driver.run("select * from old_hbase_hive_table");
+
+ driver.run("drop table old_hbase_hive_table");
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeDesc.java
index 3e850f0..ba99d3b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeDesc.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeDesc.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.ddl.table.create.like;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
+import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTLT;
import java.io.Serializable;
import java.util.HashMap;
@@ -118,6 +119,7 @@ public class CreateTableLikeDesc implements DDLDesc,
Serializable {
public Map<String, String> getTblPropsExplain() {
HashMap<String, String> copy = new HashMap<>(tblProps);
copy.remove(TABLE_IS_CTAS);
+ copy.remove(TABLE_IS_CTLT);
return copy;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index d179a94..f0d5747 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13678,6 +13678,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
tblProps = validateAndAddDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols,
isMaterialization, isTemporary,
isTransactional, isManaged);
+ tblProps.put(hive_metastoreConstants.TABLE_IS_CTLT, "true");
addDbAndTabToOutputs(new String[] {qualifiedTabName.getDb(),
qualifiedTabName.getTable()},
TableType.MANAGED_TABLE, isTemporary, tblProps, storageFormat);
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
index 1b981f5..097ed89 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
@@ -75,6 +75,8 @@ hive_metastoreConstants::hive_metastoreConstants() {
TABLE_IS_CTAS = "created_with_ctas";
+ TABLE_IS_CTLT = "created_with_ctlt";
+
PARTITION_TRANSFORM_SPEC = "partition_transform_spec";
NO_CLEANUP = "no_cleanup";
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h
index d5f7cdf..807e777 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h
@@ -47,6 +47,7 @@ class hive_metastoreConstants {
std::string DRUID_CONFIG_PREFIX;
std::string JDBC_CONFIG_PREFIX;
std::string TABLE_IS_CTAS;
+ std::string TABLE_IS_CTLT;
std::string PARTITION_TRANSFORM_SPEC;
std::string NO_CLEANUP;
std::string CTAS_LEGACY_CONFIG;
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
index 2851b22..8576117 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
@@ -73,6 +73,8 @@ package org.apache.hadoop.hive.metastore.api;
public static final java.lang.String TABLE_IS_CTAS = "created_with_ctas";
+ public static final java.lang.String TABLE_IS_CTLT = "created_with_ctlt";
+
public static final java.lang.String PARTITION_TRANSFORM_SPEC =
"partition_transform_spec";
public static final java.lang.String NO_CLEANUP = "no_cleanup";
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php
index b6f4e80..9210480 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php
@@ -50,6 +50,7 @@ final class Constant extends \Thrift\Type\TConstant
static protected $DRUID_CONFIG_PREFIX;
static protected $JDBC_CONFIG_PREFIX;
static protected $TABLE_IS_CTAS;
+ static protected $TABLE_IS_CTLT;
static protected $PARTITION_TRANSFORM_SPEC;
static protected $NO_CLEANUP;
static protected $CTAS_LEGACY_CONFIG;
@@ -214,6 +215,11 @@ final class Constant extends \Thrift\Type\TConstant
return "created_with_ctas";
}
+ protected static function init_TABLE_IS_CTLT()
+ {
+ return "created_with_ctlt";
+ }
+
protected static function init_PARTITION_TRANSFORM_SPEC()
{
return "partition_transform_spec";
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py
index ef5a2b1..f582bdc 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py
@@ -44,6 +44,7 @@ TABLE_BUCKETING_VERSION = "bucketing_version"
DRUID_CONFIG_PREFIX = "druid."
JDBC_CONFIG_PREFIX = "hive.sql."
TABLE_IS_CTAS = "created_with_ctas"
+TABLE_IS_CTLT = "created_with_ctlt"
PARTITION_TRANSFORM_SPEC = "partition_transform_spec"
NO_CLEANUP = "no_cleanup"
CTAS_LEGACY_CONFIG = "create_table_as_external"
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb
b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb
index d441f3a..b9b4ce8 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb
@@ -71,6 +71,8 @@ JDBC_CONFIG_PREFIX = %q"hive.sql."
TABLE_IS_CTAS = %q"created_with_ctas"
+TABLE_IS_CTLT = %q"created_with_ctlt"
+
PARTITION_TRANSFORM_SPEC = %q"partition_transform_spec"
NO_CLEANUP = %q"no_cleanup"
diff --git
a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index a03e3c9..f5dfa54 100644
---
a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++
b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -3136,6 +3136,7 @@ const string TABLE_BUCKETING_VERSION =
"bucketing_version",
const string DRUID_CONFIG_PREFIX = "druid.",
const string JDBC_CONFIG_PREFIX = "hive.sql.",
const string TABLE_IS_CTAS = "created_with_ctas",
+const string TABLE_IS_CTLT = "created_with_ctlt",
const string PARTITION_TRANSFORM_SPEC = "partition_transform_spec",
const string NO_CLEANUP = "no_cleanup",
const string CTAS_LEGACY_CONFIG = "create_table_as_external",
\ No newline at end of file
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 83a838f..6e8b85d 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -121,6 +121,7 @@ import static
org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import static
org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import static
org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedTableName;
+import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTLT;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
@@ -2283,6 +2284,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
if (tbl.getParameters() != null) {
tbl.getParameters().remove(TABLE_IS_CTAS);
+ tbl.getParameters().remove(TABLE_IS_CTLT);
}
// If the given table has column statistics, save it here. We will update
it later.
