This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 41340eca3cf branch-3.1: [fix](schema-cache) make catalog level schema
cache config work for "get_schema_from_table" mode #51057 (#52221)
41340eca3cf is described below
commit 41340eca3cffb3f6483b022245337571da504d17
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Tue Jun 24 20:35:51 2025 +0800
branch-3.1: [fix](schema-cache) make catalog level schema cache config work
for "get_schema_from_table" mode #51057 (#52221)
bp #51057
---
.../doris/datasource/hive/HMSExternalTable.java | 11 ++--
.../datasource/hive/HiveMetaStoreClientHelper.java | 5 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
.../hive/test_hive_meta_cache.out | Bin 1649 -> 2098 bytes
.../hive/test_hive_meta_cache.groovy | 60 +++++++++++++++++++++
5 files changed, 72 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index f2097a0bf01..2f5deb79ca1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -618,7 +618,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
List<FieldSchema> schema = null;
Map<String, String> colDefaultValues = Maps.newHashMap();
if (getFromTable) {
- schema = getSchemaFromRemoteTable(remoteTable);
+ schema = getSchemaFromRemoteTable();
} else {
HMSCachedClient client = ((HMSExternalCatalog)
catalog).getClient();
schema = client.getSchema(dbName, name);
@@ -636,10 +636,13 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
return Optional.of(new HMSSchemaCacheValue(columns, partitionColumns));
}
- private static List<FieldSchema> getSchemaFromRemoteTable(Table table) {
+ private List<FieldSchema> getSchemaFromRemoteTable() {
+ // Here we should get a new remote table instead of using
this.remoteTable
+ // Because we need to get the latest schema from HMS.
+ Table newTable = ((HMSExternalCatalog)
catalog).getClient().getTable(dbName, name);
List<FieldSchema> schema = Lists.newArrayList();
- schema.addAll(table.getSd().getCols());
- schema.addAll(table.getPartitionKeys());
+ schema.addAll(newTable.getSd().getCols());
+ schema.addAll(newTable.getPartitionKeys());
return schema;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 5b810d57a42..74bf07fff1a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -713,7 +713,10 @@ public class HiveMetaStoreClientHelper {
return Type.UNSUPPORTED;
}
- public static String
showCreateTable(org.apache.hadoop.hive.metastore.api.Table remoteTable) {
+ public static String showCreateTable(HMSExternalTable hmsTable) {
+ // Always use the latest schema
+ HMSExternalCatalog catalog = (HMSExternalCatalog)
hmsTable.getCatalog();
+ Table remoteTable = catalog.getClient().getTable(hmsTable.getDbName(),
hmsTable.getRemoteName());
StringBuilder output = new StringBuilder();
if (remoteTable.isSetViewOriginalText() ||
remoteTable.isSetViewExpandedText()) {
output.append(String.format("CREATE VIEW `%s` AS ",
remoteTable.getTableName()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index b54549af69b..0ec652fd305 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1157,7 +1157,7 @@ public class ShowExecutor {
try {
if (table.getType() == TableType.HMS_EXTERNAL_TABLE) {
rows.add(Arrays.asList(table.getName(),
-
HiveMetaStoreClientHelper.showCreateTable(((HMSExternalTable)
table).getRemoteTable())));
+
HiveMetaStoreClientHelper.showCreateTable((HMSExternalTable) table)));
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
return;
}
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
index ac5d540cdce..7e031ce471d 100644
Binary files
a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out and
b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out differ
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
index 113f90707e9..aa5ba31af17 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
@@ -296,6 +296,66 @@ suite("test_hive_meta_cache",
"p0,external,hive,external_docker,external_docker_
// desc table, 6 columns
qt_sql_6col "desc test_hive_meta_cache_db.sales";
sql """drop table test_hive_meta_cache_db.sales"""
+
+ // test schema cache with get_schema_from_table
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ // 1. create catalog with schema cache off and
get_schema_from_table
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'schema.cache.ttl-second' = '0',
+ 'get_schema_from_table' = 'true'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ // desc table, 3 columns
+ qt_sql_3col "desc test_hive_meta_cache_db.sales";
+ // show create table , 3 columns
+ def sql_sct01_3col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_3col}"
+ assertTrue(sql_sct01_3col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double)\nPARTITIONED BY (\n `year` int)"));
+
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k1 string)"
+ // desc table, 4 columns
+ qt_sql_4col "desc test_hive_meta_cache_db.sales";
+ // show create table, 4 columns
+ def sql_sct01_4col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_4col}"
+ assertTrue(sql_sct01_4col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double,\n `k1` string)\nPARTITIONED BY (\n `year`
int)"));
+
+ // open schema cache
+ sql """alter catalog ${catalog_name_no_cache} set properties
("schema.cache.ttl-second" = "120")"""
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k2 string)"
+ // desc table, 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+ // show create table, 5 columns
+ def sql_sct01_5col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_5col}"
+ assertTrue(sql_sct01_5col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double,\n `k1` string,\n `k2` string)\nPARTITIONED BY
(\n `year` int)"));
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k3 string)"
+ // desc table, still 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+ // show create table always see latest schema, 6 columns
+ def sql_sct01_6col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_6col}"
+ assertTrue(sql_sct01_6col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double,\n `k1` string,\n `k2` string,\n `k3`
string)\nPARTITIONED BY (\n `year` int)"));
+ sql """drop table test_hive_meta_cache_db.sales"""
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
