This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b514d23eb57 branch-3.1: [fix](catalog) fix behavior of metadata sync
between FE #53371 (#53581)
b514d23eb57 is described below
commit b514d23eb57eb5c2fa51faf30a81379e4c948be9
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sun Jul 20 19:42:18 2025 -0700
branch-3.1: [fix](catalog) fix behavior of metadata sync between FE #53371
(#53581)
bp #53371
---
.../doris/catalog/MysqlCompatibleDatabase.java | 5 -
.../org/apache/doris/catalog/RefreshManager.java | 26 ++---
.../apache/doris/datasource/ExternalCatalog.java | 53 ++++++++-
.../apache/doris/datasource/ExternalDatabase.java | 130 ++++++++++++++-------
.../apache/doris/datasource/ExternalObjectLog.java | 3 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 19 ---
.../doris/datasource/hive/HiveMetadataOps.java | 21 ++--
.../hive/event/MetastoreEventsProcessor.java | 2 +-
.../datasource/iceberg/IcebergMetadataCache.java | 18 +++
.../datasource/iceberg/IcebergMetadataOps.java | 12 +-
.../doris/datasource/metacache/MetaCache.java | 22 ++++
.../apache/doris/persist/TruncateTableInfo.java | 2 +-
.../main/java/org/apache/doris/qe/DdlExecutor.java | 3 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 4 +
.../doris/datasource/RefreshCatalogTest.java | 6 +
.../hive/test_hive_case_sensibility.out | Bin 4125 -> 4398 bytes
.../hive/test_hive_use_meta_cache.out | Bin 2089 -> 2147 bytes
.../hive/test_hive_use_meta_cache_false.out | Bin 0 -> 1387 bytes
.../hive/test_hive_use_meta_cache_true.out | Bin 0 -> 1415 bytes
.../test_meta_cache_select_without_refresh.out | Bin 340 -> 361 bytes
.../hive/test_hive_case_sensibility.groovy | 4 +
...roovy => test_hive_use_meta_cache_false.groovy} | 40 +++++--
...groovy => test_hive_use_meta_cache_true.groovy} | 19 +--
.../test_meta_cache_select_without_refresh.groovy | 6 +-
.../tvf/test_file_tvf_hdfs.groovy | 10 +-
25 files changed, 267 insertions(+), 138 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
index 8bbdfe1d5fd..fd6fab1ab4e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
@@ -41,11 +41,6 @@ public abstract class MysqlCompatibleDatabase extends
Database {
*/
protected abstract void initTables();
- @Override
- public void unregisterTable(String name) {
- // Do nothing
- }
-
/**
* MysqlCompatibleDatabase will not be persisted to bdb.
* It will be constructed everytime the fe starts. See
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index ab0f093988d..685a1baee9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -17,7 +17,6 @@
package org.apache.doris.catalog;
-import org.apache.doris.analysis.RefreshCatalogStmt;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.UserException;
@@ -30,7 +29,6 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.persist.OperationType;
-import org.apache.doris.qe.DdlExecutor;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
@@ -74,13 +72,13 @@ public class RefreshManager {
private void refreshCatalogInternal(CatalogIf catalog, boolean
invalidCache) {
String catalogName = catalog.getName();
if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
- ((ExternalCatalog) catalog).onRefreshCache(invalidCache);
+ ((ExternalCatalog) catalog).resetToUninitialized(invalidCache);
LOG.info("refresh catalog {} with invalidCache {}", catalogName,
invalidCache);
}
}
// Refresh database
- public void handleRefreshDb(String catalogName, String dbName, boolean
invalidCache) throws DdlException {
+ public void handleRefreshDb(String catalogName, String dbName) throws
DdlException {
Env env = Env.getCurrentEnv();
CatalogIf catalog = catalogName != null ?
env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
if (catalog == null) {
@@ -90,9 +88,9 @@ public class RefreshManager {
throw new DdlException("Only support refresh database in external
catalog");
}
DatabaseIf db = catalog.getDbOrDdlException(dbName);
- refreshDbInternal((ExternalDatabase) db, invalidCache);
+ refreshDbInternal((ExternalDatabase) db);
- ExternalObjectLog log =
ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName(),
invalidCache);
+ ExternalObjectLog log =
ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName());
Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log);
}
@@ -111,17 +109,13 @@ public class RefreshManager {
if (!db.isPresent()) {
LOG.warn("failed to find db when replaying refresh db: {}",
log.debugForRefreshDb());
} else {
- refreshDbInternal(db.get(), log.isInvalidCache());
+ refreshDbInternal(db.get());
}
}
- private void refreshDbInternal(ExternalDatabase db, boolean invalidCache) {
- db.setUnInitialized();
- if (invalidCache) {
-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(db.getCatalog().getId(),
db.getFullName());
- }
- LOG.info("refresh database {} in catalog {} with invalidCache {}",
db.getFullName(),
- db.getCatalog().getName(), invalidCache);
+ private void refreshDbInternal(ExternalDatabase db) {
+ db.resetToUninitialized();
+ LOG.info("refresh database {} in catalog {}", db.getFullName(),
db.getCatalog().getName());
}
// Refresh table
@@ -287,12 +281,12 @@ public class RefreshManager {
* {@link
org.apache.doris.analysis.RefreshCatalogStmt#analyze(Analyzer)} is ok,
* because the default value of invalidCache is true.
* */
- RefreshCatalogStmt refreshCatalogStmt = new
RefreshCatalogStmt(catalogName, null);
try {
- DdlExecutor.execute(Env.getCurrentEnv(),
refreshCatalogStmt);
+
Env.getCurrentEnv().getRefreshManager().handleRefreshCatalog(catalogName, true);
} catch (Exception e) {
LOG.warn("failed to refresh catalog {}",
catalogName, e);
}
+
// reset
timeGroup[1] = original;
refreshMap.put(catalogId, timeGroup);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index f02d77e7e68..89585ece459 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -344,6 +344,9 @@ public abstract class ExternalCatalog
private void buildMetaCache() {
if (metaCache == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("buildMetaCache for catalog: {}:{}", this.name,
this.id, new Exception());
+ }
metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
name,
OptionalLong.of(Config.external_cache_expire_time_seconds_after_access),
@@ -353,7 +356,7 @@ public abstract class ExternalCatalog
localDbName -> Optional.ofNullable(
buildDbForInit(null, localDbName,
Util.genIdByName(name, localDbName), logType,
true)),
- (key, value, cause) -> value.ifPresent(v ->
v.setUnInitialized()));
+ (key, value, cause) -> value.ifPresent(v ->
v.resetToUninitialized()));
}
}
@@ -582,8 +585,9 @@ public abstract class ExternalCatalog
refreshOnlyCatalogCache(invalidCache);
}
- public void onRefreshCache(boolean invalidCache) {
- refreshOnlyCatalogCache(invalidCache);
+ // Only for hms event handling.
+ public void onRefreshCache() {
+ refreshOnlyCatalogCache(true);
}
private void refreshOnlyCatalogCache(boolean invalidCache) {
@@ -593,7 +597,7 @@ public abstract class ExternalCatalog
} else if (!useMetaCache.get()) {
this.initialized = false;
for (ExternalDatabase<? extends ExternalTable> db :
idToDb.values()) {
- db.setUnInitialized();
+ db.resetToUninitialized();
}
}
}
@@ -806,7 +810,7 @@ public abstract class ExternalCatalog
Env.getCurrentEnv().getAccessManager().removeAccessController(name);
}
- public void replayInitCatalog(InitCatalogLog log) {
+ public synchronized void replayInitCatalog(InitCatalogLog log) {
// If the remote name is missing during upgrade, or
// the refresh db's remote name is empty,
// all databases in the Map will be reinitialized.
@@ -898,6 +902,10 @@ public abstract class ExternalCatalog
public Optional<ExternalDatabase<? extends ExternalTable>>
getDbForReplay(String dbName) {
Preconditions.checkState(useMetaCache.isPresent(), name);
if (useMetaCache.get()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getDbForReplay from metacache, db: {}.{}, catalog
id: {}, is catalog init: {}",
+ this.name, dbName, this.id, isInitialized());
+ }
if (!isInitialized()) {
return Optional.empty();
}
@@ -1176,8 +1184,29 @@ public abstract class ExternalCatalog
}
}
+ /**
+ * Unregisters a database from the catalog.
+ * Internally, remove the database meta from cache
+ *
+ * @param dbName
+ */
public void unregisterDatabase(String dbName) {
- throw new NotImplementedException("unregisterDatabase not
implemented");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("unregister database [{}]", dbName);
+ }
+ if (useMetaCache.get()) {
+ if (isInitialized()) {
+ metaCache.invalidate(dbName, Util.genIdByName(name, dbName));
+ }
+ } else {
+ Long dbId = dbNameToId.remove(dbName);
+ if (dbId == null) {
+ LOG.warn("unregister database {}.{} failed, not find in map.
ignore.", this.name, dbName);
+ } else {
+ idToDb.remove(dbId);
+ }
+ }
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getId(),
dbName);
}
public void registerDatabase(long dbId, String dbName) {
@@ -1448,5 +1477,17 @@ public abstract class ExternalCatalog
throw e;
}
}
+
+ /**
+ * Resets the name list in meta cache.
+ * Usually used after creating database in catalog, so that user can see
newly created db immediately.
+ */
+ public void resetMetaCacheNames() {
+ if (useMetaCache.isPresent() && useMetaCache.get() && metaCache !=
null) {
+ metaCache.resetNames();
+ } else {
+ resetToUninitialized(true);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index aae378b75e7..ef6a42ac49d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -55,6 +55,7 @@ import com.google.gson.internal.LinkedTreeMap;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
import java.io.DataOutput;
@@ -134,7 +135,11 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
}
- public synchronized void setUnInitialized() {
+ public synchronized void resetToUninitialized() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("resetToUninitialized db name {}, id {}, isInitializing:
{}, initialized: {}",
+ this.name, this.id, isInitializing, initialized, new
Exception());
+ }
this.initialized = false;
this.lowerCaseToTableName = Maps.newConcurrentMap();
if (extCatalog.getUseMetaCache().isPresent()) {
@@ -146,6 +151,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
}
}
+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getCatalog().getId(),
getFullName());
}
public boolean isInitialized() {
@@ -181,28 +187,15 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
initialized = true;
}
+ } catch (Exception e) {
+ LOG.warn("failed to init db {}, id {}, isInitializing: {},
initialized: {}",
+ this.name, this.id, isInitializing, initialized, e);
+ initialized = false;
} finally {
isInitializing = false;
}
}
- private void buildMetaCache() {
- if (metaCache == null) {
- metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
- name,
-
OptionalLong.of(Config.external_cache_expire_time_seconds_after_access),
- OptionalLong.of(Config.external_cache_refresh_time_minutes
* 60L),
- Config.max_meta_object_cache_num,
- ignored -> listTableNames(),
- localTableName -> Optional.ofNullable(
- buildTableForInit(null, localTableName,
- Util.genIdByName(extCatalog.getName(),
name, localTableName),
- extCatalog,
- this, true)),
- (key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
- }
- }
-
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
// If the remote name is missing during upgrade, all tables in the Map
will be reinitialized.
if ((log.getCreateCount() > 0 && (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty()))
@@ -313,6 +306,26 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog);
}
+ private void buildMetaCache() {
+ if (metaCache == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("buildMetaCache for database: {}:{}", this.name,
this.id, new Exception());
+ }
+ metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
+ name,
+
OptionalLong.of(Config.external_cache_expire_time_seconds_after_access),
+ OptionalLong.of(Config.external_cache_refresh_time_minutes
* 60L),
+ Config.max_meta_object_cache_num,
+ ignored -> listTableNames(),
+ localTableName -> Optional.ofNullable(
+ buildTableForInit(null, localTableName,
+ Util.genIdByName(extCatalog.getName(),
name, localTableName),
+ extCatalog,
+ this, true)),
+ (key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
+ }
+ }
+
private List<Pair<String, String>> listTableNames() {
List<Pair<String, String>> tableNames;
lowerCaseToTableName.clear();
@@ -388,12 +401,12 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
try {
List<String> tblNames =
Lists.newArrayList(getTableNamesWithLock());
if (!tblNames.contains(localTableName)) {
- tblNames = listTableNames().stream()
- .map(Pair::value)
- .collect(Collectors.toList());
+ // reset the table name list to ensure it is up-to-date
+ resetMetaCacheNames();
+ tblNames = Lists.newArrayList(getTableNamesWithLock());
if (!tblNames.contains(localTableName)) {
- LOG.warn("Table {} does not exist in the remote
system. Skipping initialization.",
- localTableName);
+ LOG.warn("Table {} does not exist in the remote
database {}. Skipping initialization.",
+ localTableName, this.name);
return null;
}
}
@@ -468,13 +481,21 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
*/
public Optional<T> getTableForReplay(String tblName) {
Preconditions.checkState(extCatalog.getUseMetaCache().isPresent(),
extCatalog.getName() + "." + name);
+ String localName = getLocalTableName(tblName, true);
+ if (localName == null) {
+ return Optional.empty();
+ }
if (extCatalog.getUseMetaCache().get()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getTableForReplay from metacache, db: {}.{}, is db
init: {}",
+ this.extCatalog.getName(), this.name, isInitialized());
+ }
if (!isInitialized()) {
return Optional.empty();
}
- return metaCache.tryGetMetaObj(tblName);
- } else if (tableNameToId.containsKey(tblName)) {
- return
Optional.ofNullable(idToTbl.get(tableNameToId.get(tblName)));
+ return metaCache.tryGetMetaObj(localName);
+ } else if (tableNameToId.containsKey(localName)) {
+ return
Optional.ofNullable(idToTbl.get(tableNameToId.get(localName)));
} else {
return Optional.empty();
}
@@ -615,6 +636,32 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
@Override
public T getTableNullable(String tableName) {
makeSureInitialized();
+ String finalName = getLocalTableName(tableName, false);
+ if (finalName == null) {
+ return null;
+ }
+ if (extCatalog.getUseMetaCache().get()) {
+ // must use full qualified name to generate id.
+ // otherwise, if 2 databases have the same table name, the id will
be the same.
+ return metaCache.getMetaObj(finalName,
+ Util.genIdByName(extCatalog.getName(), name,
finalName)).orElse(null);
+ } else {
+ if (!tableNameToId.containsKey(finalName)) {
+ return null;
+ }
+ return idToTbl.get(tableNameToId.get(finalName));
+ }
+ }
+
+ /**
+ * Get the local table name based on the given table name.
+ *
+ * @param tableName
+ * @param isReplay, if true, only check local cache, will not visit remote
system
+ * @return
+ */
+ @Nullable
+ private String getLocalTableName(String tableName, boolean isReplay) {
String finalName = tableName;
if (this.isStoredTableNamesLowerCase()) {
finalName = tableName.toLowerCase();
@@ -622,13 +669,20 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
if (this.isTableNamesCaseInsensitive()) {
finalName = lowerCaseToTableName.get(tableName.toLowerCase());
if (finalName == null) {
+ if (isReplay) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("failed to get final table name from:
{}.{}.{}, is replay = true",
+ getCatalog().getName(), getFullName(),
tableName);
+ }
+ return null;
+ }
// Here we need to execute listTableNames() once to fill in
lowerCaseToTableName
// to prevent lowerCaseToTableName from being empty in some
cases
listTableNames();
finalName = lowerCaseToTableName.get(tableName.toLowerCase());
if (finalName == null) {
if (LOG.isDebugEnabled()) {
- LOG.info("failed to get final table name from:
{}.{}.{}",
+ LOG.debug("failed to get final table name from:
{}.{}.{}",
getCatalog().getName(), getFullName(),
tableName);
}
return null;
@@ -643,17 +697,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
LOG.debug("get table {} from database: {}.{}, final name is: {},
catalog id: {}",
tableName, getCatalog().getName(), getFullName(),
finalName, getCatalog().getId());
}
- if (extCatalog.getUseMetaCache().get()) {
- // must use full qualified name to generate id.
- // otherwise, if 2 databases have the same table name, the id will
be the same.
- return metaCache.getMetaObj(finalName,
- Util.genIdByName(extCatalog.getName(), name,
finalName)).orElse(null);
- } else {
- if (!tableNameToId.containsKey(finalName)) {
- return null;
- }
- return idToTbl.get(tableNameToId.get(finalName));
- }
+ return finalName;
}
@Override
@@ -746,7 +790,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public void unregisterTable(String tableName) {
makeSureInitialized();
if (LOG.isDebugEnabled()) {
- LOG.debug("try unregister table [{}]", tableName);
+ LOG.debug("unregister table {}.{}", this.name, tableName);
}
setLastUpdateTime(System.currentTimeMillis());
// check if the table exists in cache, it not, does return
@@ -847,4 +891,12 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
buildMetaCache();
metaCache.addObjForTest(tbl.getId(), tbl.getName(), tbl);
}
+
+ public void resetMetaCacheNames() {
+ if (extCatalog.getUseMetaCache().isPresent() &&
extCatalog.getUseMetaCache().get() && metaCache != null) {
+ metaCache.resetNames();
+ } else {
+ resetToUninitialized();
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
index 04d90c9a049..411d40c07d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
@@ -62,11 +62,10 @@ public class ExternalObjectLog implements Writable {
}
- public static ExternalObjectLog createForRefreshDb(long catalogId, String
dbName, boolean invalidCache) {
+ public static ExternalObjectLog createForRefreshDb(long catalogId, String
dbName) {
ExternalObjectLog externalObjectLog = new ExternalObjectLog();
externalObjectLog.setCatalogId(catalogId);
externalObjectLog.setDbName(dbName);
- externalObjectLog.setInvalidCache(invalidCache);
return externalObjectLog;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 502c579fa0e..804e95b1ecd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -227,25 +227,6 @@ public class HMSExternalCatalog extends ExternalCatalog {
return ((HiveMetadataOps) metadataOps).getClient();
}
- @Override
- public void unregisterDatabase(String dbName) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("drop database [{}]", dbName);
- }
- if (useMetaCache.get()) {
- if (isInitialized()) {
- metaCache.invalidate(dbName, Util.genIdByName(name, dbName));
- }
- } else {
- Long dbId = dbNameToId.remove(dbName);
- if (dbId == null) {
- LOG.warn("drop database [{}] failed", dbName);
- }
- idToDb.remove(dbId);
- }
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getId(),
dbName);
- }
-
@Override
public void registerDatabase(long dbId, String dbName) {
if (LOG.isDebugEnabled()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index 8ea7b2fb27a..163cbcc51ce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -126,7 +126,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
@Override
public void afterCreateDb() {
- catalog.onRefreshCache(true);
+ catalog.resetMetaCacheNames();
}
@Override
@@ -167,7 +167,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
@Override
public void afterDropDb(String dbName) {
- catalog.onRefreshCache(true);
+ catalog.unregisterDatabase(dbName);
}
@Override
@@ -289,7 +289,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
public void afterCreateTable(String dbName, String tblName) {
Optional<ExternalDatabase<?>> db = catalog.getDbForReplay(dbName);
if (db.isPresent()) {
- db.get().setUnInitialized();
+ db.get().resetMetaCacheNames();
}
LOG.info("after create table {}.{}.{}, is db exists: {}",
getCatalog().getName(), dbName, tblName, db.isPresent());
@@ -321,11 +321,7 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
public void afterDropTable(String dbName, String tblName) {
Optional<ExternalDatabase<?>> db = catalog.getDbForReplay(dbName);
if (db.isPresent()) {
- Optional table = db.get().getTableForReplay(tblName);
- if (table.isPresent()) {
-
Env.getCurrentEnv().getRefreshManager().refreshTableInternal(db.get(),
(ExternalTable) table.get(), 0);
- }
- db.get().setUnInitialized();
+ db.get().unregisterTable(tblName);
}
LOG.info("after drop table {}.{}.{}, is db exists: {}",
getCatalog().getName(), dbName, tblName, db.isPresent());
@@ -347,12 +343,11 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
// Invalidate cache.
Optional<ExternalDatabase<?>> db = catalog.getDbForReplay(dbName);
if (db.isPresent()) {
- Optional dorisTable = db.get().getTableForReplay(tblName);
- if (dorisTable.isPresent()) {
-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache((ExternalTable)
dorisTable.get());
+ Optional tbl = db.get().getTableForReplay(tblName);
+ if (tbl.isPresent()) {
+ Env.getCurrentEnv().getRefreshManager()
+ .refreshTableInternal(db.get(), (ExternalTable)
tbl.get(), 0);
}
- db.get().setLastUpdateTime(System.currentTimeMillis());
- db.get().setUnInitialized();
}
} catch (Exception e) {
LOG.warn("exception when calling afterTruncateTable for db: {},
table: {}, error: {}",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
index 23bf324360b..f053ad5d46a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
@@ -128,7 +128,7 @@ public class MetastoreEventsProcessor extends MasterDaemon {
} catch (MetastoreNotificationFetchException e) {
LOG.warn("Failed to fetch hms events on {}. msg: ",
hmsExternalCatalog.getName(), e);
} catch (Exception ex) {
- hmsExternalCatalog.onRefreshCache(true);
+ hmsExternalCatalog.onRefreshCache();
updateLastSyncedEventId(hmsExternalCatalog, -1);
LOG.warn("Failed to process hive metastore [{}] events .",
hmsExternalCatalog.getName(), ex);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
index 01e37a58520..f79c28b39b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
@@ -39,6 +39,8 @@ import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
import org.apache.iceberg.view.View;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import java.util.HashMap;
@@ -48,6 +50,7 @@ import java.util.OptionalLong;
import java.util.concurrent.ExecutorService;
public class IcebergMetadataCache {
+ private static final Logger LOG =
LogManager.getLogger(IcebergMetadataCache.class);
private final LoadingCache<IcebergMetadataCacheKey, List<Snapshot>>
snapshotListCache;
private final LoadingCache<IcebergMetadataCacheKey, Table> tableCache;
@@ -120,6 +123,9 @@ public class IcebergMetadataCache {
throw new RuntimeException("Only support 'hms' and 'iceberg' type
for iceberg table");
}
try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("load iceberg table {}", nameMapping, new
Exception());
+ }
return ((ExternalCatalog)
catalog).getPreExecutionAuthenticator().execute(()
-> ops.loadTable(nameMapping.getRemoteDbName(),
nameMapping.getRemoteTblName()));
} catch (Exception e) {
@@ -160,6 +166,10 @@ public class IcebergMetadataCache {
.filter(entry -> entry.getKey().nameMapping.getCtlId() ==
catalogId)
.forEach(entry -> {
ManifestFiles.dropCache(entry.getValue().io());
+ if (LOG.isDebugEnabled()) {
+ LOG.info("invalidate iceberg table cache {} when
invalidating catalog cache",
+ entry.getKey().nameMapping, new Exception());
+ }
tableCache.invalidate(entry.getKey());
});
@@ -191,6 +201,10 @@ public class IcebergMetadataCache {
})
.forEach(entry -> {
ManifestFiles.dropCache(entry.getValue().io());
+ if (LOG.isDebugEnabled()) {
+ LOG.info("invalidate iceberg table cache {}",
+ entry.getKey().nameMapping, new Exception());
+ }
tableCache.invalidate(entry.getKey());
});
@@ -223,6 +237,10 @@ public class IcebergMetadataCache {
})
.forEach(entry -> {
ManifestFiles.dropCache(entry.getValue().io());
+ if (LOG.isDebugEnabled()) {
+ LOG.info("invalidate iceberg table cache {} when
invalidating db cache",
+ entry.getKey().nameMapping, new Exception());
+ }
tableCache.invalidate(entry.getKey());
});
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 5c34201b009..98c7c8048f0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -176,7 +176,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
@Override
public void afterCreateDb() {
- dorisCatalog.onRefreshCache(true);
+ dorisCatalog.resetMetaCacheNames();
}
private boolean performCreateDb(String dbName, boolean ifNotExists,
Map<String, String> properties)
@@ -248,7 +248,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
@Override
public void afterDropDb(String dbName) {
- dorisCatalog.onRefreshCache(true);
+ dorisCatalog.unregisterDatabase(dbName);
}
@Override
@@ -312,7 +312,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
public void afterCreateTable(String dbName, String tblName) {
Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForReplay(dbName);
if (db.isPresent()) {
- db.get().setUnInitialized();
+ db.get().resetMetaCacheNames();
}
LOG.info("after create table {}.{}.{}, is db exists: {}",
dorisCatalog.getName(), dbName, tblName, db.isPresent());
@@ -340,11 +340,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
public void afterDropTable(String dbName, String tblName) {
Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForReplay(dbName);
if (db.isPresent()) {
- Optional table = db.get().getTableForReplay(tblName);
- if (table.isPresent()) {
-
Env.getCurrentEnv().getRefreshManager().refreshTableInternal(db.get(),
(ExternalTable) table.get(), 0);
- }
- db.get().setUnInitialized();
+ db.get().unregisterTable(tblName);
}
LOG.info("after drop table {}.{}.{}. is db exists: {}",
dorisCatalog.getName(), dbName, tblName, db.isPresent());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
index 3cc47d26d3d..a98dbc132e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -26,6 +26,8 @@ import com.github.benmanes.caffeine.cache.RemovalListener;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Map;
@@ -36,6 +38,7 @@ import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
public class MetaCache<T> {
+ private static final Logger LOG = LogManager.getLogger(MetaCache.class);
private LoadingCache<String, List<Pair<String, String>>> namesCache;
//Pair<String, String> : <Remote name, Local name>
private Map<Long, String> idToName = Maps.newConcurrentMap();
@@ -95,6 +98,10 @@ public class MetaCache<T> {
if (val != null && val.isPresent()) {
return val;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("trigger getMetaObj in metacache {}, obj name:
{}, id: {}",
+ this.name, name, id, new Exception());
+ }
metaObjCache.invalidate(name);
val = metaObjCache.get(name);
idToName.put(id, name);
@@ -138,12 +145,19 @@ public class MetaCache<T> {
return v;
}
});
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("invalidate obj in metacache {}, obj name: {}, id: {}",
+ name, localName, id, new Exception());
+ }
metaObjCache.invalidate(localName);
idToName.remove(id);
}
public void invalidateAll() {
namesCache.invalidateAll();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("invalidate all in metacache {}", name, new Exception());
+ }
metaObjCache.invalidateAll();
idToName.clear();
}
@@ -158,4 +172,12 @@ public class MetaCache<T> {
idToName.put(id, name);
metaObjCache.put(name, Optional.of(db));
}
+
+ /**
+ * Reset the names cache.
+ * Should only be used after creating new database/table
+ */
+ public void resetNames() {
+ namesCache.invalidateAll();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
index c9b000297db..be7df995919 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
@@ -141,7 +141,7 @@ public class TruncateTableInfo implements Writable {
public String toString() {
return "TruncateTableInfo{"
+ "ctl=" + ctl
- + "dbId=" + dbId
+ + ", dbId=" + dbId
+ ", db='" + db + '\''
+ ", tblId=" + tblId
+ ", table='" + table + '\''
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index 472826450e8..93b187a0f60 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -350,8 +350,7 @@ public class DdlExecutor {
refreshTableStmt.getTblName(), false);
} else if (ddlStmt instanceof RefreshDbStmt) {
RefreshDbStmt refreshDbStmt = (RefreshDbStmt) ddlStmt;
-
env.getRefreshManager().handleRefreshDb(refreshDbStmt.getCatalogName(),
refreshDbStmt.getDbName(),
- refreshDbStmt.isInvalidCache());
+
env.getRefreshManager().handleRefreshDb(refreshDbStmt.getCatalogName(),
refreshDbStmt.getDbName());
} else if (ddlStmt instanceof AlterResourceStmt) {
env.getResourceMgr().alterResource((AlterResourceStmt) ddlStmt);
} else if (ddlStmt instanceof AlterColocateGroupStmt) {
diff --git
a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 5c86d9025e8..30af287bca1 100644
---
a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -643,6 +643,10 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
isConnected = false;
TTransportException tte = null;
MetaException lastException = null;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("HiveMetaStoreClient open() called with conf: " +
conf.toString());
+ }
boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
boolean useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL);
boolean useFramedTransport = MetastoreConf.getBoolVar(conf,
ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 39a6394477e..6329de3c886 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -148,6 +148,7 @@ public class RefreshCatalogTest extends TestWithFeService {
// when use_meta_cache is true, the table will be recreated after
refresh.
// so we need to get table again
table = (TestExternalTable)
test2.getDbNullable("db1").getTable("tbl11").get();
+ Assertions.assertTrue(((ExternalCatalog) test2).isInitialized());
Assertions.assertFalse(table.isObjectCreated());
test2.getDbNullable("db1").getTables();
Assertions.assertFalse(table.isObjectCreated());
@@ -156,6 +157,10 @@ public class RefreshCatalogTest extends TestWithFeService {
} catch (Exception e) {
// Do nothing
}
+ // after refresh, the catalog will be set to uninitialized
+ Assertions.assertFalse(((ExternalCatalog) test2).isInitialized());
+ // call get table to trigger catalog initialization
+ table = (TestExternalTable)
test2.getDbNullable("db1").getTable("tbl11").get();
Assertions.assertTrue(((ExternalCatalog) test2).isInitialized());
}
@@ -231,3 +236,4 @@ public class RefreshCatalogTest extends TestWithFeService {
}
}
}
+
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out
b/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out
index 8270e6e2bde..5ce72113334 100644
Binary files
a/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out
and
b/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out
differ
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
index 7f28eae6135..3a4fbd9190a 100644
Binary files
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out and
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
differ
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache_false.out
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache_false.out
new file mode 100644
index 00000000000..3df89cd3b33
Binary files /dev/null and
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache_false.out
differ
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache_true.out
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache_true.out
new file mode 100644
index 00000000000..5c0701b852a
Binary files /dev/null and
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache_true.out
differ
diff --git
a/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out
b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out
index 7669a50d30c..af3b52782d0 100644
Binary files
a/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out
and
b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out
differ
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy
index caea986adec..bdf8ee094ed 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy
@@ -267,7 +267,11 @@ suite("test_hive_case_sensibility",
"p0,external,doris,external_docker,external_
} else {
sql """truncate table CASE_TBL12;"""
}
+ // before truncate, see in show tables
+ qt_before_truncate_show_table """show tables like
"%case_tbl12%""""
sql """truncate table case_tbl12;"""
+ // after truncate, should also see in show tables
+ qt_after_truncate_show_table """show tables like
"%case_tbl12%""""
qt_sql20 """select * from case_tbl12;""" // empty
// 6. drop table
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache_false.groovy
similarity index 85%
copy from
regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
copy to
regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache_false.groovy
index 76b19cf6b9e..cd5abf4fcba 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache_false.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_docker_hive") {
+suite("test_hive_use_meta_cache_false",
"p0,external,hive,external_docker,external_docker_hive") {
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -23,6 +23,27 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
return;
}
+ def allFrontends = sql """show frontends;"""
+ logger.info("allFrontends: " + allFrontends)
+ def frontendCounts = allFrontends.size()
+ def isMaster = true
+
+ for (def i = 0; i < frontendCounts; i++) {
+ def currentFrontend = allFrontends[i]
+ def isCurrent = currentFrontend[18]
+ if (isCurrent.equals("Yes")) {
+ isMaster = (currentFrontend[8].equals("true"))
+ break
+ }
+ }
+
+ logger.info("connect to master:: " + isMaster)
+ if (!isMaster) {
+ // use_meta_cache = false with connection to non master
+ // will cause the result unstable, so skip it
+ return;
+ }
+
for (String hivePrefix : ["hive2", "hive3"]) {
setHivePrefix(hivePrefix)
try {
@@ -42,11 +63,11 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
);"""
// create from Doris, the cache will be filled immediately
- String database= "test_use_meta_cache_db"
- String table = "test_use_meta_cache_tbl"
- String database_hive = "test_use_meta_cache_db_hive"
- String table_hive = "test_use_meta_cache_tbl_hive"
- String partitioned_table_hive =
"test_use_meta_cache_partitioned_tbl_hive"
+ String database= "test_use_meta_cache_db_${use_meta_cache}"
+ String table = "test_use_meta_cache_tbl_${use_meta_cache}"
+ String database_hive =
"test_use_meta_cache_db_hive_${use_meta_cache}"
+ String table_hive =
"test_use_meta_cache_tbl_hive_${use_meta_cache}"
+ String partitioned_table_hive =
"test_use_meta_cache_partitioned_tbl_hive_${use_meta_cache}"
sql "switch ${catalog}"
sql "drop database if exists ${database} force"
@@ -76,6 +97,7 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
} else {
// if not use meta cache, can not use
sql "refresh catalog ${catalog}"
+ sql "show databases"
sql "use ${database_hive}"
}
@@ -89,13 +111,14 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
if (use_meta_cache) {
// but can select
sql "select * from ${table_hive}"
- // still not see
+ // after select, the names cache is refresh, so can see
order_qt_sql06 "show tables"
sql "refresh database ${database_hive}"
} else {
// if not use meta cache, can not select
sql "refresh database ${database_hive}"
sql "select * from ${table_hive}"
+ // after select, the names cache is refresh, so can see
order_qt_sql06 "show tables"
}
// can see
@@ -128,6 +151,8 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
// can see because partition file listing is not cached
order_qt_sql09 "show partitions from ${partitioned_table_hive}"
+ // before drop table, we can see the tables
+ order_qt_sql091 "show tables"
// drop tables
hive_docker "drop table
${database_hive}.${partitioned_table_hive}"
hive_docker "drop table ${database_hive}.${table_hive}"
@@ -145,7 +170,6 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
// can not see
order_qt_sql13 "show databases like '%${database_hive}%'";
}
- test_use_meta_cache(true)
test_use_meta_cache(false)
} finally {
}
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache_true.groovy
similarity index 92%
rename from
regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
rename to
regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache_true.groovy
index 76b19cf6b9e..7bf9eee738f 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache_true.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_docker_hive") {
+suite("test_hive_use_meta_cache_true",
"p0,external,hive,external_docker,external_docker_hive") {
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -42,11 +42,11 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
);"""
// create from Doris, the cache will be filled immediately
- String database= "test_use_meta_cache_db"
- String table = "test_use_meta_cache_tbl"
- String database_hive = "test_use_meta_cache_db_hive"
- String table_hive = "test_use_meta_cache_tbl_hive"
- String partitioned_table_hive =
"test_use_meta_cache_partitioned_tbl_hive"
+ String database= "test_use_meta_cache_db_${use_meta_cache}"
+ String table = "test_use_meta_cache_tbl_${use_meta_cache}"
+ String database_hive =
"test_use_meta_cache_db_hive_${use_meta_cache}"
+ String table_hive =
"test_use_meta_cache_tbl_hive_${use_meta_cache}"
+ String partitioned_table_hive =
"test_use_meta_cache_partitioned_tbl_hive_${use_meta_cache}"
sql "switch ${catalog}"
sql "drop database if exists ${database} force"
@@ -76,6 +76,7 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
} else {
// if not use meta cache, can not use
sql "refresh catalog ${catalog}"
+ sql "show databases"
sql "use ${database_hive}"
}
@@ -89,13 +90,14 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
if (use_meta_cache) {
// but can select
sql "select * from ${table_hive}"
- // still not see
+ // after select, the names cache is refresh, so can see
order_qt_sql06 "show tables"
sql "refresh database ${database_hive}"
} else {
// if not use meta cache, can not select
sql "refresh database ${database_hive}"
sql "select * from ${table_hive}"
+ // after select, the names cache is refresh, so can see
order_qt_sql06 "show tables"
}
// can see
@@ -128,6 +130,8 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
// can see because partition file listing is not cached
order_qt_sql09 "show partitions from ${partitioned_table_hive}"
+ // before drop table, we can see the tables
+ order_qt_sql091 "show tables"
// drop tables
hive_docker "drop table
${database_hive}.${partitioned_table_hive}"
hive_docker "drop table ${database_hive}.${table_hive}"
@@ -146,7 +150,6 @@ suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_doc
order_qt_sql13 "show databases like '%${database_hive}%'";
}
test_use_meta_cache(true)
- test_use_meta_cache(false)
} finally {
}
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
index c2073f9864c..0be1ea92563 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
@@ -91,11 +91,7 @@ suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_dock
sql """insert into internal.external_lower_select_without_refresh.TABLE2
values(1, 'TABLE2')"""
- test {
- sql """select * from
test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table2;"""
-
- exception "Table [table2] does not exist in database
[external_lower_select_without_refresh]."
- }
+ qt_sql """select * from
test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table2;"""
qt_test_meta_cache_lower_false_select_without_refresh_select_table2
"select * from
test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.TABLE2;"
diff --git
a/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
b/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
index d5d9f3ca032..b2704a2cf46 100644
--- a/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
@@ -333,9 +333,9 @@
suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") {
// test create view from tvf and alter view from tvf
uri = "${defaultFS}" +
"/user/doris/preinstalled_data/csv_format_test/all_types.csv"
format = "csv"
- sql """ DROP VIEW IF EXISTS test_hdfs_tvf_create_view;"""
+ sql """ DROP VIEW IF EXISTS test_file_tvf_hdfs_create_view;"""
sql """
- create view test_hdfs_tvf_create_view as
+ create view test_file_tvf_hdfs_create_view as
select * from FILE(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
@@ -343,10 +343,10 @@
suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") {
"format" = "${format}") order by c1;
"""
- order_qt_create_view """ select * from test_hdfs_tvf_create_view
order by c1 limit 20; """
+ order_qt_create_view """ select * from
test_file_tvf_hdfs_create_view order by c1 limit 20; """
sql """
- alter view test_hdfs_tvf_create_view as
+ alter view test_file_tvf_hdfs_create_view as
select c1 from FILE(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
@@ -354,7 +354,7 @@
suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") {
"format" = "${format}") order by c1;
"""
- order_qt_alter_view """ select * from test_hdfs_tvf_create_view
order by c1 limit 20; """
+ order_qt_alter_view """ select * from
test_file_tvf_hdfs_create_view order by c1 limit 20; """
} finally {
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
