This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new f2556ba1823 [feature](insert)support external hive truncate table DDL
(#37659)
f2556ba1823 is described below
commit f2556ba1823fa2a1f79253151703a42cdb272d39
Author: slothever <[email protected]>
AuthorDate: Fri Jul 12 22:37:47 2024 +0800
[feature](insert)support external hive truncate table DDL (#37659)
pick: #36801
---
.../apache/doris/analysis/TruncateTableStmt.java | 3 -
.../main/java/org/apache/doris/catalog/Env.java | 6 +-
.../org/apache/doris/datasource/CatalogIf.java | 3 +
.../apache/doris/datasource/ExternalCatalog.java | 23 +++++
.../doris/datasource/hive/HMSCachedClient.java | 2 +
.../doris/datasource/hive/HiveMetadataOps.java | 16 ++++
.../hive/PostgreSQLJdbcHMSCachedClient.java | 4 +
.../datasource/hive/ThriftHMSCachedClient.java | 19 +++-
.../datasource/iceberg/IcebergMetadataOps.java | 5 ++
.../datasource/operations/ExternalMetadataOps.java | 8 ++
.../doris/datasource/TestHMSCachedClient.java | 3 +
.../hive/ddl/test_hive_truncate_table.out | 24 +++++
.../hive/ddl/test_hive_truncate_table.groovy | 100 +++++++++++++++++++++
13 files changed, 210 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
index a275879692c..1a9fbd3bafd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
@@ -23,7 +23,6 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.InternalDatabaseUtil;
-import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -44,8 +43,6 @@ public class TruncateTableStmt extends DdlStmt {
public void analyze(Analyzer analyzer) throws AnalysisException,
UserException {
super.analyze(analyzer);
tblRef.getName().analyze(analyzer);
- // disallow external catalog
- Util.prohibitExternalCatalog(tblRef.getName().getCtl(),
this.getClass().getSimpleName());
if (tblRef.hasExplicitAlias()) {
throw new AnalysisException("Not support truncate table with
alias");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index ea8ffd91d2e..9b73a4517f0 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -5427,8 +5427,10 @@ public class Env {
* otherwise, it will only truncate those specified partitions.
*
*/
- public void truncateTable(TruncateTableStmt truncateTableStmt) throws
DdlException {
- getInternalCatalog().truncateTable(truncateTableStmt);
+ public void truncateTable(TruncateTableStmt stmt) throws DdlException {
+ CatalogIf<?> catalogIf =
catalogMgr.getCatalogOrException(stmt.getTblRef().getName().getCtl(),
+ catalog -> new DdlException(("Unknown catalog " + catalog)));
+ catalogIf.truncateTable(stmt);
}
public void replayTruncateTable(TruncateTableInfo info) throws
MetaNotFoundException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index ceee1a68157..a79897c67df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
@@ -194,4 +195,6 @@ public interface CatalogIf<T extends DatabaseIf> {
boolean createTable(CreateTableStmt stmt) throws UserException;
void dropTable(DropTableStmt stmt) throws DdlException;
+
+ void truncateTable(TruncateTableStmt truncateTableStmt) throws
DdlException;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 0fbedf57e66..82f4d309c82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -22,6 +22,8 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TableRef;
+import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
@@ -821,4 +823,25 @@ public abstract class ExternalCatalog
}
return ret;
}
+
+ @Override
+ public void truncateTable(TruncateTableStmt stmt) throws DdlException {
+ makeSureInitialized();
+ if (metadataOps == null) {
+ throw new UnsupportedOperationException("Truncate table not
supported in " + getName());
+ }
+ try {
+ TableRef tableRef = stmt.getTblRef();
+ TableName tableName = tableRef.getName();
+ // delete all table data if null
+ List<String> partitions = null;
+ if (tableRef.getPartitionNames() != null) {
+ partitions = tableRef.getPartitionNames().getPartitionNames();
+ }
+ metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(),
partitions);
+ } catch (Exception e) {
+ LOG.warn("Failed to drop a table", e);
+ throw e;
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
index c9d0ce1736b..1ac77972053 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
@@ -94,6 +94,8 @@ public interface HMSCachedClient {
void dropTable(String dbName, String tableName);
+ void truncateTable(String dbName, String tblName, List<String> partitions);
+
void createTable(TableMetadata catalogTable, boolean ignoreIfExists);
void updateTableStatistics(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index 70c61875b8a..7267297c93e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -251,6 +251,22 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
}
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String>
partitions) throws DdlException {
+ ExternalDatabase<?> db = catalog.getDbNullable(dbName);
+ if (db == null) {
+ throw new DdlException("Failed to get database: '" + dbName + "'
in catalog: " + catalog.getName());
+ }
+ try {
+ client.truncateTable(dbName, tblName, partitions);
+ } catch (Exception e) {
+ throw new DdlException(e.getMessage(), e);
+ }
+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(),
dbName, tblName);
+ db.setLastUpdateTime(System.currentTimeMillis());
+ db.setUnInitialized(true);
+ }
+
@Override
public List<String> listTableNames(String dbName) {
return client.getAllTables(dbName);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
index 8e41b48bfdd..0cdb3e469c2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
@@ -546,6 +546,10 @@ public class PostgreSQLJdbcHMSCachedClient extends
JdbcHMSCachedClient {
throw new NotImplementedException("PostgreSQL dropDatabase not
implemented");
}
+ public void truncateTable(String dbName, String tblName, List<String>
partitions) {
+ throw new NotImplementedException("PostgreSQL truncateTable not
implemented");
+ }
+
public void createTable(TableMetadata hiveTable, boolean ignoreIfExists) {
throw new NotImplementedException("PostgreSQL createTable not
implemented");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
index 17fbcb09b02..55d8ffc2e02 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
@@ -244,6 +244,23 @@ public class ThriftHMSCachedClient implements
HMSCachedClient {
}
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String>
partitions) {
+ try (ThriftHMSClient client = getClient()) {
+ try {
+ ugiDoAs(() -> {
+ client.client.truncateTable(dbName, tblName, partitions);
+ return null;
+ });
+ } catch (Exception e) {
+ client.setThrowable(e);
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new HMSClientException("failed to truncate table %s in db
%s.", e, tblName, dbName);
+ }
+ }
+
@Override
public boolean tableExists(String dbName, String tblName) {
try (ThriftHMSClient client = getClient()) {
@@ -272,7 +289,7 @@ public class ThriftHMSCachedClient implements
HMSCachedClient {
throw e;
}
} catch (Exception e) {
- throw new HMSClientException("failed to check if table %s in db %s
exists", e, tblName, dbName);
+ throw new HMSClientException("failed to list partitions in table
'%s.%s'.", e, dbName, tblName);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 7161f48680a..c7fef68ee97 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -173,4 +173,9 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
catalog.dropTable(TableIdentifier.of(dbName, tableName));
db.setUnInitialized(true);
}
+
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String>
partitions) {
+ throw new UnsupportedOperationException("Truncate Iceberg table is not
supported.");
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
index 9426442cebb..0333124b352 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
@@ -60,6 +60,14 @@ public interface ExternalMetadataOps {
*/
void dropTable(DropTableStmt stmt) throws DdlException;
+ /**
+ *
+ * @param dbName
+ * @param tblName
+ * @param partitions
+ */
+ void truncateTable(String dbName, String tblName, List<String> partitions)
throws DdlException;
+
/**
*
* @return
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
index 6f969257245..f3cb918d6f5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
@@ -235,6 +235,9 @@ public class TestHMSCachedClient implements HMSCachedClient
{
this.partitions.remove(new HMSTransaction.DatabaseTableName(dbName,
tableName));
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String>
partitions) {}
+
@Override
public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
String dbName = tbl.getDbName();
diff --git
a/regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
b/regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
new file mode 100644
index 00000000000..24d6cade5d2
--- /dev/null
+++
b/regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !truncate_01_pre --
+222 aoe
+3234424 44
+
+-- !truncate_01 --
+
+-- !truncate_02 --
+
+-- !truncate_03 --
+
+-- !truncate_04_pre --
+33 awe wuu 2023-02-04
+5535 3 dre 2023-04-04
+
+-- !truncate_04 --
+
+-- !truncate_06 --
+44 etg wuweu 2022-02-04
+555 etgf wet 2021-05-06
+88 etg wuweu 2022-01-04
+95 etgf hiyr 2021-05-06
+
+-- !truncate_09 --
diff --git
a/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
b/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
new file mode 100644
index 00000000000..94082d0287f
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_truncate_table",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String hms_port = context.config.otherConfigs.get("hive3HmsPort")
+ String hdfs_port = context.config.otherConfigs.get("hive3HdfsPort")
+ String catalog_name = "test_hive3_truncate_table"
+ sql """drop catalog if exists ${catalog_name};"""
+
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hms_port}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'use_meta_cache' = 'true',
+ 'hive.version'='3.0'
+ );
+ """
+
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """create database if not exists `hive_truncate`;"""
+ sql """use `hive_truncate`;"""
+
+ // 1. test no partition table
+ sql """create table if not exists `table_no_pars`(col1 bigint, col2
string) engine=hive; """
+ sql """truncate table table_no_pars;"""
+
+ sql """insert into `table_no_pars` values(3234424, '44'); """
+ sql """insert into `table_no_pars` values(222, 'aoe'); """
+ order_qt_truncate_01_pre """ select * from table_no_pars; """
+ sql """truncate table table_no_pars;"""
+ order_qt_truncate_01 """ select * from table_no_pars; """
+
+ sql """insert into `table_no_pars` values(3234424, '44'); """
+ sql """truncate table hive_truncate.table_no_pars;"""
+ order_qt_truncate_02 """ select * from table_no_pars; """
+
+ sql """insert into `table_no_pars` values(222, 'aoe'); """
+ sql """truncate table ${catalog_name}.hive_truncate.table_no_pars;"""
+ order_qt_truncate_03 """ select * from table_no_pars; """
+ sql """insert into `table_no_pars` values(222, 'aoe'); """
+ sql """drop table table_no_pars """
+
+ // 2. test partition table
+ sql """create table if not exists `table_with_pars`(col1 bigint, col2
string, pt1 varchar, pt2 date) engine=hive
+ partition by list(pt1, pt2)() """
+ sql """truncate table table_with_pars;"""
+
+ sql """insert into `table_with_pars` values(33, 'awe', 'wuu',
'2023-02-04') """
+ sql """insert into `table_with_pars` values(5535, '3', 'dre',
'2023-04-04') """
+ order_qt_truncate_04_pre """ select * from table_with_pars; """
+ sql """truncate table table_with_pars;"""
+ order_qt_truncate_04 """ select * from table_with_pars; """
+
+ // 3. test partition table and truncate partitions
+ sql """insert into `table_with_pars` values(44, 'etg', 'wuweu',
'2022-02-04') """
+ sql """insert into `table_with_pars` values(88, 'etg', 'wuweu',
'2022-01-04') """
+ sql """insert into `table_with_pars` values(095, 'etgf', 'hiyr',
'2021-05-06') """
+ sql """insert into `table_with_pars` values(555, 'etgf', 'wet',
'2021-05-06') """
+ // sql """truncate table hive_truncate.table_with_pars partition
pt1;"""
+ // order_qt_truncate_05 """ select * from table_with_pars; """
+ // sql """truncate table hive_truncate.table_with_pars partition
pt2;"""
+ order_qt_truncate_06 """ select * from table_with_pars; """
+
+ sql """insert into `table_with_pars` values(22, 'ttt', 'gggw',
'2022-02-04')"""
+ sql """insert into `table_with_pars` values(44, 'etg', 'wuweu',
'2022-02-04') """
+ sql """insert into `table_with_pars` values(88, 'etg', 'wuweu',
'2022-01-04') """
+ sql """insert into `table_with_pars` values(095, 'etgf', 'hiyr',
'2021-05-06') """
+ sql """insert into `table_with_pars` values(555, 'etgf', 'wet',
'2021-05-06') """
+ // sql """truncate table ${catalog_name}.hive_truncate.table_with_pars
partition pt1;"""
+ // order_qt_truncate_07 """ select * from table_with_pars; """
+ // sql """truncate table ${catalog_name}.hive_truncate.table_with_pars
partition pt2;"""
+ // order_qt_truncate_08 """ select * from table_with_pars; """
+ sql """truncate table table_with_pars"""
+ order_qt_truncate_09 """ select * from table_with_pars; """
+
+ sql """drop table table_with_pars """
+ sql """drop database hive_truncate;"""
+ sql """drop catalog ${catalog_name};"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
