This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2d8b0a31f46 HIVE-26474: Enable converting insert-only tables to full
ACID tables (Simhadri Govindappa, reviewed by Denys Kuzmenko)
2d8b0a31f46 is described below
commit 2d8b0a31f461a33ef2e19c0269ec44778d5ca136
Author: SimhadriGovindappa <[email protected]>
AuthorDate: Thu Sep 1 16:22:52 2022 +0530
HIVE-26474: Enable converting insert-only tables to full ACID tables
(Simhadri Govindappa, reviewed by Denys Kuzmenko)
Closes #3550
---
.../org/apache/hadoop/hive/conf/Constants.java | 4 +
.../AlterTableSetPropertiesOperation.java | 13 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 132 +++----
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +-
.../apache/hadoop/hive/ql/util/UpgradeTool.java | 35 +-
.../clientpositive/insert_only_to_acid_convert.q | 31 ++
.../llap/insert_only_to_acid_convert.q.out | 394 +++++++++++++++++++++
.../metastore/TransactionalValidationListener.java | 4 +-
.../apache/hadoop/hive/metastore/txn/TxnUtils.java | 27 +-
9 files changed, 526 insertions(+), 117 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java
b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
index b89cdf3fadc..bd489f84400 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
@@ -93,4 +93,8 @@ public class Constants {
public static final String INSERT_ONLY_FETCH_BUCKET_ID =
"insertonly.fetch.bucketid";
public static final String ERROR_MESSAGE_NO_DETAILS_AVAILABLE = "No detailed
message available";
+
+ public static final String ORC_INPUT_FORMAT =
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
+ public static final String ORC_OUTPUT_FORMAT =
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat";
+
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/properties/AlterTableSetPropertiesOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/properties/AlterTableSetPropertiesOperation.java
index fb29d07054c..ee53595e975 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/properties/AlterTableSetPropertiesOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/properties/AlterTableSetPropertiesOperation.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation;
@@ -43,6 +44,8 @@ import org.apache.hadoop.hive.ql.plan.MoveWork;
import com.google.common.collect.Lists;
+import static
org.apache.hadoop.hive.metastore.TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY;
+
/**
* Operation process of setting properties of a table.
*/
@@ -62,6 +65,7 @@ public class AlterTableSetPropertiesOperation extends
AbstractAlterTableOperatio
} else {
boolean isFromMmTable =
AcidUtils.isInsertOnlyTable(table.getParameters());
Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(table,
desc.getProps());
+ boolean isToFullAcid = AcidUtils.isToFullAcid(table, desc.getProps());
if (!isFromMmTable && BooleanUtils.isTrue(isToMmTable)) {
if (!HiveConf.getBoolVar(context.getConf(),
ConfVars.HIVE_MM_ALLOW_ORIGINALS)) {
List<Task<?>> mmTasks = generateAddMmTasks(table, desc.getWriteId());
@@ -79,8 +83,13 @@ public class AlterTableSetPropertiesOperation extends
AbstractAlterTableOperatio
checkMmLb(table);
}
}
- } else if (isFromMmTable && BooleanUtils.isFalse(isToMmTable)) {
- throw new HiveException("Cannot convert an ACID table to non-ACID");
+ } else if (isFromMmTable) {
+ if (isToFullAcid) {
+
table.getParameters().put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
+ DEFAULT_TRANSACTIONAL_PROPERTY);
+ } else if (BooleanUtils.isFalse(isToMmTable)) {
+ throw new HiveException("Cannot convert an ACID table to non-ACID");
+ }
}
// Converting to/from external table
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index a9c3bb5a827..2811763c062 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -21,7 +21,10 @@ package org.apache.hadoop.hive.ql.io;
import static
org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_PATH_SUFFIX;
import static org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_TABLE;
import static org.apache.hadoop.hive.common.FileUtils.HIDDEN_FILES_PATH_FILTER;
+import static
org.apache.hadoop.hive.metastore.TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY;
+import static
org.apache.hadoop.hive.metastore.TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY;
import static org.apache.hadoop.hive.ql.exec.Utilities.COPY_KEYWORD;
+
import static org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
import java.io.FileNotFoundException;
@@ -55,6 +58,7 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
@@ -74,6 +78,7 @@ import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.LockComponentBuilder;
@@ -82,6 +87,7 @@ import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.TxnType;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.Context;
@@ -474,11 +480,7 @@ public class AcidUtils {
* @return true, if the tblProperties contains {@link
AcidUtils#COMPACTOR_TABLE_PROPERTY}
*/
public static boolean isCompactionTable(Properties tblProperties) {
- if (tblProperties != null &&
tblProperties.containsKey(COMPACTOR_TABLE_PROPERTY) && tblProperties
- .getProperty(COMPACTOR_TABLE_PROPERTY).equalsIgnoreCase("true")) {
- return true;
- }
- return false;
+ return tblProperties != null &&
isCompactionTable(Maps.fromProperties(tblProperties));
}
/**
@@ -487,7 +489,7 @@ public class AcidUtils {
* @return true, if the parameters contains {@link
AcidUtils#COMPACTOR_TABLE_PROPERTY}
*/
public static boolean isCompactionTable(Map<String, String> parameters) {
- return Boolean.valueOf(parameters.getOrDefault(COMPACTOR_TABLE_PROPERTY,
"false"));
+ return Boolean.parseBoolean(parameters.get(COMPACTOR_TABLE_PROPERTY));
}
/**
@@ -1927,11 +1929,7 @@ public class AcidUtils {
}
public static boolean isTablePropertyTransactional(Properties props) {
- String resultStr =
props.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- if (resultStr == null) {
- resultStr =
props.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase());
- }
- return resultStr != null && resultStr.equalsIgnoreCase("true");
+ return isTablePropertyTransactional(Maps.fromProperties(props));
}
public static boolean isTablePropertyTransactional(Map<String, String>
parameters) {
@@ -1939,18 +1937,9 @@ public class AcidUtils {
if (resultStr == null) {
resultStr =
parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase());
}
- return resultStr != null && resultStr.equalsIgnoreCase("true");
+ return Boolean.parseBoolean(resultStr);
}
- public static boolean isTablePropertyTransactional(Configuration conf) {
- String resultStr =
conf.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- if (resultStr == null) {
- resultStr =
conf.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase());
- }
- return resultStr != null && resultStr.equalsIgnoreCase("true");
- }
-
-
/**
* @param p - not null
*/
@@ -1967,12 +1956,16 @@ public class AcidUtils {
return isTransactionalTable(table.getTblProps());
}
+ public static boolean isTransactionalTable(Table table) {
+ return table != null && isTransactionalTable(table.getTTable());
+ }
+
+ public static boolean
isTransactionalTable(org.apache.hadoop.hive.metastore.api.Table table) {
+ return table != null && isTransactionalTable(table.getParameters());
+ }
+
public static boolean isTransactionalTable(Map<String, String> props) {
- String tableIsTransactional =
props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- if (tableIsTransactional == null) {
- tableIsTransactional =
props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase());
- }
- return tableIsTransactional != null &&
tableIsTransactional.equalsIgnoreCase("true");
+ return props != null && isTablePropertyTransactional(props);
}
public static boolean isTransactionalView(CreateMaterializedViewDesc view) {
@@ -1982,16 +1975,18 @@ public class AcidUtils {
return isTransactionalTable(view.getTblProps());
}
+ public static boolean isFullAcidTable(CreateTableDesc td) {
+ if (td == null || td.getTblProps() == null) {
+ return false;
+ }
+ return isFullAcidTable(td.getTblProps());
+ }
/**
* Should produce the same result as
* {@link
org.apache.hadoop.hive.metastore.txn.TxnUtils#isAcidTable(org.apache.hadoop.hive.metastore.api.Table)}
*/
public static boolean isFullAcidTable(Table table) {
- return isFullAcidTable(table == null ? null : table.getTTable());
- }
-
- public static boolean isTransactionalTable(Table table) {
- return isTransactionalTable(table == null ? null : table.getTTable());
+ return table != null && isFullAcidTable(table.getTTable());
}
/**
@@ -1999,31 +1994,13 @@ public class AcidUtils {
* {@link
org.apache.hadoop.hive.metastore.txn.TxnUtils#isAcidTable(org.apache.hadoop.hive.metastore.api.Table)}
*/
public static boolean
isFullAcidTable(org.apache.hadoop.hive.metastore.api.Table table) {
- return isTransactionalTable(table) &&
- !isInsertOnlyTable(table.getParameters());
+ return table != null && isFullAcidTable(table.getParameters());
}
public static boolean isFullAcidTable(Map<String, String> params) {
return isTransactionalTable(params) && !isInsertOnlyTable(params);
}
- public static boolean
isTransactionalTable(org.apache.hadoop.hive.metastore.api.Table table) {
- return table != null && table.getParameters() != null &&
- isTablePropertyTransactional(table.getParameters());
- }
-
- public static boolean isFullAcidTable(CreateTableDesc td) {
- if (td == null || td.getTblProps() == null) {
- return false;
- }
- String tableIsTransactional =
td.getTblProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- if (tableIsTransactional == null) {
- tableIsTransactional =
td.getTblProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase());
- }
- return tableIsTransactional != null &&
tableIsTransactional.equalsIgnoreCase("true") &&
- !AcidUtils.isInsertOnlyTable(td.getTblProps());
- }
-
public static boolean isFullAcidScan(Configuration conf) {
if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN)) {
return false;
@@ -2200,22 +2177,16 @@ public class AcidUtils {
* @return true if table is an INSERT_ONLY table, false otherwise
*/
public static boolean isInsertOnlyTable(Map<String, String> params) {
- return isInsertOnlyTable(params, false);
+ String transactionalProp =
params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ return
INSERTONLY_TRANSACTIONAL_PROPERTY.equalsIgnoreCase(transactionalProp);
}
+
public static boolean isInsertOnlyTable(Table table) {
return isTransactionalTable(table) &&
getAcidOperationalProperties(table).isInsertOnly();
}
-
- // TODO [MM gap]: CTAS may currently be broken. It used to work. See the old
code, and why isCtas isn't used?
- public static boolean isInsertOnlyTable(Map<String, String> params, boolean
isCtas) {
- String transactionalProp =
params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- return (transactionalProp != null &&
"insert_only".equalsIgnoreCase(transactionalProp));
- }
-
+
public static boolean isInsertOnlyTable(Properties params) {
- String transactionalProp = params.getProperty(
- hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- return (transactionalProp != null &&
"insert_only".equalsIgnoreCase(transactionalProp));
+ return isInsertOnlyTable(Maps.fromProperties(params));
}
/**
@@ -2240,13 +2211,15 @@ public class AcidUtils {
if (transactional == null && tbl != null) {
transactional =
tbl.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
}
- boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
+ boolean isSetToTxn = Boolean.parseBoolean(transactional);
if (transactionalProp == null) {
if (isSetToTxn || tbl == null) return false; // Assume the full ACID
table.
throw new RuntimeException("Cannot change '" +
hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
+ "' without '" +
hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
}
- if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; //
Not MM.
+ if
(!INSERTONLY_TRANSACTIONAL_PROPERTY.equalsIgnoreCase(transactionalProp)) {
+ return false; // Not MM.
+ }
if (!isSetToTxn) {
if (tbl == null) return true; // No table information yet; looks like it
could be valid.
throw new RuntimeException("Cannot set '"
@@ -2256,6 +2229,39 @@ public class AcidUtils {
return true;
}
+ public static Boolean isToFullAcid(Table table, Map<String, String> props) {
+ if (AcidUtils.isTransactionalTable(table)) {
+ String transactionalProp =
props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+
+ if (DEFAULT_TRANSACTIONAL_PROPERTY.equalsIgnoreCase(transactionalProp)) {
+ return canBeMadeAcid(table.getTableName(), table.getSd());
+ }
+ }
+ return false;
+ }
+
+ public static boolean canBeMadeAcid(String fullTableName, StorageDescriptor
sd) {
+ return isAcidInputOutputFormat(fullTableName, sd) && sd.getSortColsSize()
<= 0;
+ }
+
+ private static boolean isAcidInputOutputFormat(String fullTableName,
StorageDescriptor sd) {
+ if (sd.getInputFormat() == null || sd.getOutputFormat() == null) {
+ return false;
+ }
+ try {
+ return Class.forName(Constants.ORC_INPUT_FORMAT)
+ .isAssignableFrom(Class.forName(sd.getInputFormat()))
+ && Class.forName(Constants.ORC_OUTPUT_FORMAT)
+ .isAssignableFrom(Class.forName(sd.getOutputFormat()));
+
+ } catch (ClassNotFoundException e) {
+ //if a table is using some custom I/O format and it's not in the
classpath, we won't mark
+ //the table for Acid, but today OrcInput/OutputFormat is the only Acid
format
+ LOG.error("Could not determine if " + fullTableName + " can be made Acid
due to: " + e.getMessage(), e);
+ return false;
+ }
+ }
+
public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
boolean hasTxn =
removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
hasProps =
removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index c3e7fb4a200..9b6c995bbd1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2456,8 +2456,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
}
try {
CreateTableDesc tblDesc = qb.getTableDesc();
- if (tblDesc != null && tblDesc.isTemporary() && AcidUtils
- .isInsertOnlyTable(tblDesc.getTblProps(), true)) {
+ if (tblDesc != null && tblDesc.isTemporary() &&
AcidUtils.isInsertOnlyTable(tblDesc.getTblProps())) {
fname = FileUtils.makeQualified(location, conf).toString();
} else {
fname =
ctx.getExtTmpPathRelTo(FileUtils.makeQualified(location, conf)).toString();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/UpgradeTool.java
b/ql/src/java/org/apache/hadoop/hive/ql/util/UpgradeTool.java
index e9e49a62701..67df6a7bcec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/UpgradeTool.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/UpgradeTool.java
@@ -31,10 +31,8 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
@@ -43,18 +41,14 @@ import
org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.utils.FileUtils;
import
org.apache.hadoop.hive.metastore.utils.FileUtils.RemoteIteratorWithFilter;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.BucketCodec;
-import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.thrift.TException;
@@ -524,7 +518,7 @@ public class UpgradeTool {
* ORC uses table props for settings so things like bucketing, I/O Format,
etc should
* be the same for each partition.
*/
- boolean canBeMadeAcid = canBeMadeAcid(fullTableName, t.getSd());
+ boolean canBeMadeAcid = AcidUtils.canBeMadeAcid(fullTableName, t.getSd());
if(t.getPartitionKeysSize() <= 0) {
if(canBeMadeAcid) {
convertToAcid.add("ALTER TABLE " + Warehouse.getQualifiedName(t) + "
SET TBLPROPERTIES (" +
@@ -584,33 +578,8 @@ public class UpgradeTool {
}
}
}
- private static boolean canBeMadeAcid(String fullTableName, StorageDescriptor
sd) {
- return isAcidInputOutputFormat(fullTableName, sd) && sd.getSortColsSize()
<= 0;
- }
- private static boolean isAcidInputOutputFormat(String fullTableName,
StorageDescriptor sd) {
- try {
- Class inputFormatClass = sd.getInputFormat() == null ? null :
- Class.forName(sd.getInputFormat());
- Class outputFormatClass = sd.getOutputFormat() == null ? null :
- Class.forName(sd.getOutputFormat());
- if (inputFormatClass != null && outputFormatClass != null &&
- Class.forName("org.apache.hadoop.hive.ql.io.AcidInputFormat")
- .isAssignableFrom(inputFormatClass) &&
- Class.forName("org.apache.hadoop.hive.ql.io.AcidOutputFormat")
- .isAssignableFrom(outputFormatClass)) {
- return true;
- }
- } catch (ClassNotFoundException e) {
- //if a table is using some custom I/O format and it's not in the
classpath, we won't mark
- //the table for Acid, but today (Hive 3.1 and earlier)
OrcInput/OutputFormat is the only
- //Acid format
- LOG.error("Could not determine if " + fullTableName +
- " can be made Acid due to: " + e.getMessage(), e);
- return false;
- }
- return false;
- }
+
private static void makeConvertTableScript(List<String> alterTableAcid,
List<String> alterTableMm,
String scriptLocation) throws IOException {
if (alterTableAcid.isEmpty()) {
diff --git a/ql/src/test/queries/clientpositive/insert_only_to_acid_convert.q
b/ql/src/test/queries/clientpositive/insert_only_to_acid_convert.q
new file mode 100644
index 00000000000..61ee7e41a13
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_only_to_acid_convert.q
@@ -0,0 +1,31 @@
+--! qt:dataset:alltypesorc
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.mapred.mode=nonstrict;
+set hive.strict.checks.bucketing=true;
+set hive.enforce.bucketing=true;
+
+create table insert_only(col1 Int, col2 String) stored as orc TBLPROPERTIES
('transactional'='true','transactional_properties'='insert_only');
+insert into insert_only values(1,'hi'),(2,'hello');
+describe formatted insert_only;
+ALTER TABLE insert_only SET TBLPROPERTIES
('transactional'='true','transactional_properties'='default');
+describe formatted insert_only;
+insert into insert_only values(1,'hi'),(2,'hello');
+select * from insert_only;
+
+create table insert_only_1(col1 Int, col2 String) stored as orc TBLPROPERTIES
('transactional'='true','transactional_properties'='insert_only');
+insert into insert_only_1 values(1,'hi'),(2,'hello');
+describe formatted insert_only_1;
+ALTER TABLE insert_only SET TBLPROPERTIES
('transactional_properties'='default');
+describe formatted insert_only_1;
+insert into insert_only_1 values(1,'hi'),(2,'hello');
+select * from insert_only_1;
+
+create table insert_only_part(a INT, b STRING) partitioned by (ds string)
CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES
('transactional'='true', 'transactional_properties'='insert_only');
+insert into table insert_only_part partition (ds = 'today') select cint,
cstring1 from alltypesorc where cint is not null order by cint limit 10;
+insert into table insert_only_part partition (ds = 'tomorrow') select cint,
cstring1 from alltypesorc where cint is not null order by cint limit 10;
+describe formatted insert_only_part;
+ALTER TABLE insert_only_part SET TBLPROPERTIES
('transactional'='true','transactional_properties'='default');
+describe formatted insert_only_part;
+select * from insert_only_part order by a, b, ds;
+
diff --git
a/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out
b/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out
new file mode 100644
index 00000000000..284e5d419a2
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out
@@ -0,0 +1,394 @@
+PREHOOK: query: create table insert_only(col1 Int, col2 String) stored as orc
TBLPROPERTIES ('transactional'='true','transactional_properties'='insert_only')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_only
+POSTHOOK: query: create table insert_only(col1 Int, col2 String) stored as orc
TBLPROPERTIES ('transactional'='true','transactional_properties'='insert_only')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_only
+PREHOOK: query: insert into insert_only values(1,'hi'),(2,'hello')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@insert_only
+POSTHOOK: query: insert into insert_only values(1,'hi'),(2,'hello')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@insert_only
+POSTHOOK: Lineage: insert_only.col1 SCRIPT []
+POSTHOOK: Lineage: insert_only.col2 SCRIPT []
+PREHOOK: query: describe formatted insert_only
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@insert_only
+POSTHOOK: query: describe formatted insert_only
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@insert_only
+# col_name data_type comment
+col1 int
+col2 string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
+ bucketing_version 2
+ numFiles 1
+ numRows 2
+ rawDataSize 182
+ totalSize 310
+ transactional true
+ transactional_properties insert_only
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: ALTER TABLE insert_only SET TBLPROPERTIES
('transactional'='true','transactional_properties'='default')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@insert_only
+PREHOOK: Output: default@insert_only
+POSTHOOK: query: ALTER TABLE insert_only SET TBLPROPERTIES
('transactional'='true','transactional_properties'='default')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@insert_only
+POSTHOOK: Output: default@insert_only
+PREHOOK: query: describe formatted insert_only
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@insert_only
+POSTHOOK: query: describe formatted insert_only
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@insert_only
+# col_name data_type comment
+col1 int
+col2 string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
+ bucketing_version 2
+#### A masked pattern was here ####
+ numFiles 1
+ numRows 2
+ rawDataSize 182
+ totalSize 310
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+PREHOOK: query: insert into insert_only values(1,'hi'),(2,'hello')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@insert_only
+POSTHOOK: query: insert into insert_only values(1,'hi'),(2,'hello')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@insert_only
+POSTHOOK: Lineage: insert_only.col1 SCRIPT []
+POSTHOOK: Lineage: insert_only.col2 SCRIPT []
+PREHOOK: query: select * from insert_only
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_only
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_only
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_only
+#### A masked pattern was here ####
+1 hi
+2 hello
+1 hi
+2 hello
+PREHOOK: query: create table insert_only_1(col1 Int, col2 String) stored as
orc TBLPROPERTIES
('transactional'='true','transactional_properties'='insert_only')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_only_1
+POSTHOOK: query: create table insert_only_1(col1 Int, col2 String) stored as
orc TBLPROPERTIES
('transactional'='true','transactional_properties'='insert_only')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_only_1
+PREHOOK: query: insert into insert_only_1 values(1,'hi'),(2,'hello')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@insert_only_1
+POSTHOOK: query: insert into insert_only_1 values(1,'hi'),(2,'hello')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@insert_only_1
+POSTHOOK: Lineage: insert_only_1.col1 SCRIPT []
+POSTHOOK: Lineage: insert_only_1.col2 SCRIPT []
+PREHOOK: query: describe formatted insert_only_1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@insert_only_1
+POSTHOOK: query: describe formatted insert_only_1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@insert_only_1
+# col_name data_type comment
+col1 int
+col2 string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
+ bucketing_version 2
+ numFiles 1
+ numRows 2
+ rawDataSize 182
+ totalSize 310
+ transactional true
+ transactional_properties insert_only
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: ALTER TABLE insert_only SET TBLPROPERTIES
('transactional_properties'='default')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@insert_only
+PREHOOK: Output: default@insert_only
+POSTHOOK: query: ALTER TABLE insert_only SET TBLPROPERTIES
('transactional_properties'='default')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@insert_only
+POSTHOOK: Output: default@insert_only
+PREHOOK: query: describe formatted insert_only_1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@insert_only_1
+POSTHOOK: query: describe formatted insert_only_1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@insert_only_1
+# col_name data_type comment
+col1 int
+col2 string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
+ bucketing_version 2
+ numFiles 1
+ numRows 2
+ rawDataSize 182
+ totalSize 310
+ transactional true
+ transactional_properties insert_only
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: insert into insert_only_1 values(1,'hi'),(2,'hello')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@insert_only_1
+POSTHOOK: query: insert into insert_only_1 values(1,'hi'),(2,'hello')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@insert_only_1
+POSTHOOK: Lineage: insert_only_1.col1 SCRIPT []
+POSTHOOK: Lineage: insert_only_1.col2 SCRIPT []
+PREHOOK: query: select * from insert_only_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_only_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_only_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_only_1
+#### A masked pattern was here ####
+1 hi
+2 hello
+1 hi
+2 hello
+PREHOOK: query: create table insert_only_part(a INT, b STRING) partitioned by
(ds string) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES
('transactional'='true', 'transactional_properties'='insert_only')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_only_part
+POSTHOOK: query: create table insert_only_part(a INT, b STRING) partitioned by
(ds string) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES
('transactional'='true', 'transactional_properties'='insert_only')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_only_part
+PREHOOK: query: insert into table insert_only_part partition (ds = 'today')
select cint, cstring1 from alltypesorc where cint is not null order by cint
limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@insert_only_part@ds=today
+POSTHOOK: query: insert into table insert_only_part partition (ds = 'today')
select cint, cstring1 from alltypesorc where cint is not null order by cint
limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@insert_only_part@ds=today
+POSTHOOK: Lineage: insert_only_part PARTITION(ds=today).a SIMPLE
[(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: insert_only_part PARTITION(ds=today).b SIMPLE
[(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string,
comment:null), ]
+PREHOOK: query: insert into table insert_only_part partition (ds = 'tomorrow')
select cint, cstring1 from alltypesorc where cint is not null order by cint
limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@insert_only_part@ds=tomorrow
+POSTHOOK: query: insert into table insert_only_part partition (ds =
'tomorrow') select cint, cstring1 from alltypesorc where cint is not null order
by cint limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@insert_only_part@ds=tomorrow
+POSTHOOK: Lineage: insert_only_part PARTITION(ds=tomorrow).a SIMPLE
[(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: insert_only_part PARTITION(ds=tomorrow).b SIMPLE
[(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string,
comment:null), ]
+PREHOOK: query: describe formatted insert_only_part
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@insert_only_part
+POSTHOOK: query: describe formatted insert_only_part
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@insert_only_part
+# col_name data_type comment
+a int
+b string
+
+# Partition Information
+# col_name data_type comment
+ds string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
+ numFiles 4
+ numPartitions 2
+ numRows 20
+ rawDataSize 2008
+ totalSize 1920
+ transactional true
+ transactional_properties insert_only
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [a]
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: ALTER TABLE insert_only_part SET TBLPROPERTIES
('transactional'='true','transactional_properties'='default')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@insert_only_part
+PREHOOK: Output: default@insert_only_part
+POSTHOOK: query: ALTER TABLE insert_only_part SET TBLPROPERTIES
('transactional'='true','transactional_properties'='default')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@insert_only_part
+POSTHOOK: Output: default@insert_only_part
+PREHOOK: query: describe formatted insert_only_part
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@insert_only_part
+POSTHOOK: query: describe formatted insert_only_part
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@insert_only_part
+# col_name data_type comment
+a int
+b string
+
+# Partition Information
+# col_name data_type comment
+ds string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
+#### A masked pattern was here ####
+ numFiles 4
+ numPartitions 2
+ numRows 20
+ rawDataSize 2008
+ totalSize 1920
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [a]
+Sort Columns: []
+PREHOOK: query: select * from insert_only_part order by a, b, ds
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_only_part
+PREHOOK: Input: default@insert_only_part@ds=today
+PREHOOK: Input: default@insert_only_part@ds=tomorrow
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_only_part order by a, b, ds
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_only_part
+POSTHOOK: Input: default@insert_only_part@ds=today
+POSTHOOK: Input: default@insert_only_part@ds=tomorrow
+#### A masked pattern was here ####
+-1073279343 oj1YrV5Wa today
+-1073279343 oj1YrV5Wa tomorrow
+-1073051226 A34p7oRr2WvUJNf today
+-1073051226 A34p7oRr2WvUJNf tomorrow
+-1072910839 0iqrc5 today
+-1072910839 0iqrc5 tomorrow
+-1072081801 dPkN74F7 today
+-1072081801 dPkN74F7 tomorrow
+-1072076362 2uLyD28144vklju213J1mr today
+-1072076362 2uLyD28144vklju213J1mr tomorrow
+-1071480828 aw724t8c5558x2xneC624 today
+-1071480828 aw724t8c5558x2xneC624 tomorrow
+-1071363017 Anj0oF today
+-1071363017 Anj0oF tomorrow
+-1070883071 0ruyd6Y50JpdGRf6HqD today
+-1070883071 0ruyd6Y50JpdGRf6HqD tomorrow
+-1070551679 iUR3Q today
+-1070551679 iUR3Q tomorrow
+-1069736047 k17Am8uPHWk02cEf1jet today
+-1069736047 k17Am8uPHWk02cEf1jet tomorrow
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index a5f5459b9fb..45655d83fd4 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -217,10 +217,10 @@ public final class TransactionalValidationListener
extends MetaStorePreEventList
}
}
checkSorted(newTable);
- if(TxnUtils.isAcidTable(newTable) && !TxnUtils.isAcidTable(oldTable)) {
+ if (TxnUtils.isAcidTable(newTable) &&
!TxnUtils.isTransactionalTable(oldTable)) {
/* we just made an existing table full acid which wasn't acid before and
it passed all checks
initialize the Write ID sequence so that we can handle assigning ROW_IDs
to 'original'
- files already present in the table. */
+ files already present in the table. Not needed if oldTable is
insertOnly.*/
TxnStore t = TxnUtils.getTxnStore(getConf());
//For now assume no partition may have > 10M files. Perhaps better to
count them.
t.seedWriteId(new SeedTableWriteIdsRequest(newTable.getDbName(),
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index e484a97d814..05fa4bce59d 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -25,7 +25,6 @@ import
org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.metastore.DatabaseProduct;
-import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -54,6 +53,8 @@ import java.util.stream.Collectors;
import static org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_TABLE;
import static
org.apache.hadoop.hive.metastore.DatabaseProduct.determineDatabaseProduct;
+import static
org.apache.hadoop.hive.metastore.TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY;
+import static
org.apache.hadoop.hive.metastore.TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY;
public class TxnUtils {
private static final Logger LOG = LoggerFactory.getLogger(TxnUtils.class);
@@ -145,13 +146,7 @@ public class TxnUtils {
* @return true if table is a transactional table, false otherwise
*/
public static boolean isTransactionalTable(Table table) {
- if (table == null) {
- return false;
- }
- Map<String, String> parameters = table.getParameters();
- if (parameters == null) return false;
- String tableIsTransactional =
parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- return tableIsTransactional != null &&
tableIsTransactional.equalsIgnoreCase("true");
+ return table != null && isTransactionalTable(table.getParameters());
}
public static boolean isTransactionalTable(Map<String, String> parameters) {
@@ -159,7 +154,7 @@ public class TxnUtils {
return false;
}
String tableIsTransactional =
parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- return tableIsTransactional != null &&
tableIsTransactional.equalsIgnoreCase("true");
+ return Boolean.parseBoolean(tableIsTransactional);
}
/**
@@ -167,15 +162,17 @@ public class TxnUtils {
* org.apache.hadoop.hive.ql.io.AcidUtils#isAcidTable.
*/
public static boolean isAcidTable(Table table) {
- return TxnUtils.isTransactionalTable(table) &&
-
TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY.equals(table.getParameters()
- .get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES));
+ return table != null && isAcidTable(table.getParameters());
}
public static boolean isAcidTable(Map<String, String> parameters) {
- return isTransactionalTable(parameters) &&
- TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY.
-
equals(parameters.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES));
+ return isTransactionalTable(parameters) &&
DEFAULT_TRANSACTIONAL_PROPERTY.equalsIgnoreCase(
+
parameters.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES));
+ }
+
+ public static boolean isInsertOnlyTable(Table table) {
+ return TxnUtils.isTransactionalTable(table) &&
INSERTONLY_TRANSACTIONAL_PROPERTY.equalsIgnoreCase(
+
table.getParameters().get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES));
}
public static boolean isTableSoftDeleteEnabled(Table table, boolean
isSoftDelete) {
