This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 170cc46b12 [fix](hive) check hive transactional table's file format
(#20888)
170cc46b12 is described below
commit 170cc46b12f1a1fc65e1d6b815d6aef9fd66eb60
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon Jun 19 12:33:00 2023 +0800
[fix](hive) check hive transactional table's file format (#20888)
Sometimes we meet a hive table with parameter: "transactional" = "true",
but format is parquet, which is not supported.
So we need to check the input format for transactional table.
---
.../apache/doris/catalog/external/HMSExternalTable.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 64e06783c3..fe6603c60c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -61,6 +61,7 @@ public class HMSExternalTable extends ExternalTable {
private static final Logger LOG =
LogManager.getLogger(HMSExternalTable.class);
private static final Set<String> SUPPORTED_HIVE_FILE_FORMATS;
+ private static final Set<String> SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS;
private static final String TBL_PROP_TXN_PROPERTIES =
"transactional_properties";
private static final String TBL_PROP_INSERT_ONLY = "insert_only";
@@ -70,6 +71,9 @@ public class HMSExternalTable extends ExternalTable {
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.mapred.TextInputFormat");
+
+ SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS = Sets.newHashSet();
+
SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
}
private static final Set<String> SUPPORTED_HUDI_FILE_FORMATS;
@@ -191,7 +195,15 @@ public class HMSExternalTable extends ExternalTable {
}
public boolean isHiveTransactionalTable() {
- return dlaType == DLAType.HIVE &&
AcidUtils.isTransactionalTable(remoteTable);
+ return dlaType == DLAType.HIVE &&
AcidUtils.isTransactionalTable(remoteTable)
+ && isSupportedTransactionalFileFormat();
+ }
+
+ private boolean isSupportedTransactionalFileFormat() {
+ // Sometimes we meet "transactional" = "true" but format is parquet,
which is not supported.
+ // So we need to check the input format for transactional table.
+ String inputFormatName = remoteTable.getSd().getInputFormat();
+ return inputFormatName != null &&
SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS.contains(inputFormatName);
}
public boolean isFullAcidTable() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
