This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 3ae3f9d6e1f [opt](catalog) support using loading cache for db/table
list in external catalog (#33610) (#34596)
3ae3f9d6e1f is described below
commit 3ae3f9d6e1f25fd78d7ae442b57ac29dedd201fe
Author: Mingyu Chen <[email protected]>
AuthorDate: Thu May 9 17:50:39 2024 +0800
[opt](catalog) support using loading cache for db/table list in external
catalog (#33610) (#34596)
bp #33610
---
.../main/java/org/apache/doris/common/Config.java | 2 +-
.../java/org/apache/doris/catalog/Database.java | 7 +-
.../java/org/apache/doris/catalog/DatabaseIf.java | 5 +-
.../org/apache/doris/catalog/JdbcResource.java | 22 +--
.../doris/catalog/MysqlCompatibleDatabase.java | 13 --
.../org/apache/doris/catalog/RefreshManager.java | 36 ++--
.../java/org/apache/doris/common/CacheFactory.java | 7 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 9 +
.../java/org/apache/doris/common/util/Util.java | 20 ++
.../apache/doris/datasource/CatalogFactory.java | 9 +-
.../org/apache/doris/datasource/CatalogIf.java | 3 -
.../org/apache/doris/datasource/CatalogMgr.java | 26 ++-
.../apache/doris/datasource/ExternalCatalog.java | 208 ++++++++++++++-------
.../apache/doris/datasource/ExternalDatabase.java | 168 ++++++++++++-----
.../doris/datasource/ExternalMetaCacheMgr.java | 15 ++
.../doris/datasource/ExternalSchemaCache.java | 2 +-
.../apache/doris/datasource/InternalCatalog.java | 5 -
.../doris/datasource/es/EsExternalCatalog.java | 9 +-
.../doris/datasource/es/EsExternalDatabase.java | 2 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 45 ++---
.../doris/datasource/hive/HMSExternalDatabase.java | 2 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 7 +-
.../datasource/hive/event/AlterTableEvent.java | 2 +-
.../doris/datasource/hive/event/InsertEvent.java | 2 +-
.../hudi/source/HudiCachedPartitionProcessor.java | 2 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 5 -
.../iceberg/IcebergExternalDatabase.java | 2 +-
.../datasource/iceberg/IcebergMetadataCache.java | 4 +-
.../datasource/iceberg/IcebergMetadataOps.java | 7 +-
.../infoschema/ExternalInfoSchemaDatabase.java | 2 +-
.../infoschema/ExternalMysqlDatabase.java | 2 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 15 +-
.../datasource/jdbc/JdbcExternalDatabase.java | 2 +-
.../maxcompute/MaxComputeExternalDatabase.java | 2 +-
.../doris/datasource/metacache/MetaCache.java | 111 +++++++++++
.../datasource/paimon/PaimonExternalDatabase.java | 2 +-
.../doris/datasource/test/TestExternalCatalog.java | 24 ++-
.../datasource/test/TestExternalDatabase.java | 2 +-
.../java/org/apache/doris/qe/GlobalVariable.java | 2 +
.../apache/doris/service/FrontendServiceImpl.java | 6 +-
.../apache/doris/statistics/StatisticsCleaner.java | 12 +-
.../doris/statistics/query/DataBaseStats.java | 2 +-
.../doris/analysis/AlterCatalogPropsStmtTest.java | 10 +
.../org/apache/doris/common/CacheFactoryTest.java | 6 +-
.../apache/doris/datasource/CatalogMgrTest.java | 10 +-
.../doris/datasource/RefreshCatalogTest.java | 2 +-
.../datasource/iceberg/CreateIcebergTableTest.java | 12 +-
.../datasource/property/PropertyConverterTest.java | 28 +--
.../apache/doris/external/hms/HmsCatalogTest.java | 2 +
.../org/apache/doris/qe/HmsQueryCacheTest.java | 2 +
.../hive/test_hive_use_meta_cache.out | 77 ++++++++
.../org/apache/doris/regression/suite/Suite.groovy | 1 +
.../hive/test_hive_use_meta_cache.groovy | 102 ++++++++++
.../hive/write/test_hive_write_partitions.groovy | 2 +-
54 files changed, 785 insertions(+), 299 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 826b1ab8618..9973de61be6 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1956,7 +1956,7 @@ public class Config extends ConfigBase {
* only for certain test type. E.g. only settting batch_size to small
* value for p0.
*/
- @ConfField(mutable = true, masterOnly = false)
+ @ConfField(mutable = true, masterOnly = false, options = {"p0"})
public static String fuzzy_test_type = "";
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index fd709963a7b..548119dfb71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -49,7 +49,6 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -416,6 +415,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
}
+ @Override
public boolean registerTable(TableIf table) {
boolean result = true;
Table olapTable = (Table) table;
@@ -851,11 +851,6 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
return null;
}
- @Override
- public Map<Long, TableIf> getIdToTable() {
- return new HashMap<>(idToTable);
- }
-
public void replayUpdateDbProperties(Map<String, String> properties) {
dbProperties.updateProperties(properties);
if (PropertyAnalyzer.hasBinlogConfig(properties)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
index 6c0d46ffe98..3aae592dec0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
@@ -31,7 +31,6 @@ import org.apache.logging.log4j.Logger;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -75,7 +74,7 @@ public interface DatabaseIf<T extends TableIf> {
List<T> getTables();
- default List<T> getTablesOrEmpty() {
+ default List<T> getTablesIgnoreException() {
try {
return getTables();
} catch (Exception e) {
@@ -281,6 +280,4 @@ public interface DatabaseIf<T extends TableIf> {
default long getLastUpdateTime() {
return -1L;
}
-
- Map<Long, TableIf> getIdToTable();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
index 65bf7d308ff..95522503691 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.ExternalCatalog;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -125,20 +126,8 @@ public class JdbcResource extends Resource {
CONNECTION_POOL_MAX_LIFE_TIME,
CONNECTION_POOL_MAX_WAIT_TIME,
CONNECTION_POOL_KEEP_ALIVE,
- TEST_CONNECTION
- ).build();
- private static final ImmutableList<String> OPTIONAL_PROPERTIES = new
ImmutableList.Builder<String>().add(
- ONLY_SPECIFIED_DATABASE,
- LOWER_CASE_META_NAMES,
- META_NAMES_MAPPING,
- INCLUDE_DATABASE_LIST,
- EXCLUDE_DATABASE_LIST,
- CONNECTION_POOL_MIN_SIZE,
- CONNECTION_POOL_MAX_SIZE,
- CONNECTION_POOL_MAX_LIFE_TIME,
- CONNECTION_POOL_MAX_WAIT_TIME,
- CONNECTION_POOL_KEEP_ALIVE,
- TEST_CONNECTION
+ TEST_CONNECTION,
+ ExternalCatalog.USE_META_CACHE
).build();
// The default value of optional properties
@@ -157,6 +146,8 @@ public class JdbcResource extends Resource {
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_WAIT_TIME,
"5000");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_KEEP_ALIVE,
"false");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(TEST_CONNECTION, "true");
+ OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.USE_META_CACHE,
+ String.valueOf(ExternalCatalog.DEFAULT_USE_META_CACHE));
}
// timeout for both connection and read. 10 seconds is long enough.
@@ -225,7 +216,7 @@ public class JdbcResource extends Resource {
@Override
public void applyDefaultProperties() {
- for (String s : OPTIONAL_PROPERTIES) {
+ for (String s : OPTIONAL_PROPERTIES_DEFAULT_VALUE.keySet()) {
if (!configs.containsKey(s)) {
configs.put(s, OPTIONAL_PROPERTIES_DEFAULT_VALUE.get(s));
}
@@ -487,3 +478,4 @@ public class JdbcResource extends Resource {
}
}
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
index 028fe186b2c..8bbdfe1d5fd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
@@ -17,8 +17,6 @@
package org.apache.doris.catalog;
-import org.apache.doris.alter.Alter;
-import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateViewStmt;
import org.apache.doris.common.Pair;
@@ -43,17 +41,6 @@ public abstract class MysqlCompatibleDatabase extends
Database {
*/
protected abstract void initTables();
- /**
- * Currently, rename a table of InfoSchemaDb will throw exception
- * {@link Alter#processAlterTable(AlterTableStmt)}
- * so we follow this design.
- * @note: Rename a table of mysql database in MYSQL ls allowed.
- */
- @Override
- public boolean registerTable(TableIf table) {
- return super.registerTable(table);
- }
-
@Override
public void unregisterTable(String name) {
// Do nothing
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 39efce4c70f..d017ba7829f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -41,6 +41,7 @@ import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -108,10 +109,14 @@ public class RefreshManager {
private void refreshDbInternal(long catalogId, long dbId, boolean
invalidCache) {
ExternalCatalog catalog = (ExternalCatalog)
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
- ExternalDatabase db = catalog.getDbForReplay(dbId);
- db.setUnInitialized(invalidCache);
- LOG.info("refresh database {} in catalog {} with invalidCache {}",
db.getFullName(), catalog.getName(),
- invalidCache);
+ Optional<ExternalDatabase<? extends ExternalTable>> db =
catalog.getDbForReplay(dbId);
+ // Database may not exist if 'use_meta_cache' is true.
+ // Because each FE fetch the meta data independently.
+ db.ifPresent(e -> {
+ e.setUnInitialized(invalidCache);
+ LOG.info("refresh database {} in catalog {} with invalidCache {}",
e.getFullName(),
+ catalog.getName(), invalidCache);
+ });
}
// Refresh table
@@ -163,13 +168,22 @@ public class RefreshManager {
LOG.warn("failed to find catalog replaying refresh table {}",
log.getCatalogId());
return;
}
- ExternalDatabase db = catalog.getDbForReplay(log.getDbId());
- TableIf table = db.getTableForReplay(log.getTableId());
- refreshTableInternal(catalog, db, table, log.getLastUpdateTime());
+ Optional<ExternalDatabase<? extends ExternalTable>> db =
catalog.getDbForReplay(log.getDbId());
+ // See comment in refreshDbInternal for why db and table may be null.
+ if (!db.isPresent()) {
+ LOG.warn("failed to find db replaying refresh table {}",
log.getDbId());
+ return;
+ }
+ Optional<? extends ExternalTable> table =
db.get().getTableForReplay(log.getTableId());
+ if (!table.isPresent()) {
+ LOG.warn("failed to find table replaying refresh table {}",
log.getTableId());
+ return;
+ }
+ refreshTableInternal(catalog, db.get(), table.get(),
log.getLastUpdateTime());
}
public void refreshExternalTableFromEvent(String catalogName, String
dbName, String tableName,
- long updateTime, boolean ignoreIfNotExists) throws DdlException {
+ long updateTime) throws DdlException {
CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " +
catalogName);
@@ -179,17 +193,11 @@ public class RefreshManager {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- if (!ignoreIfNotExists) {
- throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
- }
return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- if (!ignoreIfNotExists) {
- throw new DdlException("Table " + tableName + " does not exist
in db " + dbName);
- }
return;
}
refreshTableInternal(catalog, db, table, updateTime);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java
index fbb004e5c99..50f46647975 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java
@@ -22,6 +22,7 @@ import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.RemovalListener;
import com.github.benmanes.caffeine.cache.Ticker;
import org.jetbrains.annotations.NotNull;
@@ -74,9 +75,13 @@ public class CacheFactory {
}
// Build a loading cache, with executor, it will use given executor for
refresh
- public <K, V> LoadingCache<K, V> buildCache(CacheLoader<K, V> cacheLoader,
ExecutorService executor) {
+ public <K, V> LoadingCache<K, V> buildCache(CacheLoader<K, V> cacheLoader,
+ RemovalListener<K, V> removalListener, ExecutorService executor) {
Caffeine<Object, Object> builder = buildWithParams();
builder.executor(executor);
+ if (removalListener != null) {
+ builder.removalListener(removalListener);
+ }
return builder.build(cacheLoader);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index e411e992e97..ba4434349cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -35,6 +35,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.CatalogMgr;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.policy.Policy;
import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.Tag;
@@ -1348,6 +1349,14 @@ public class PropertyAnalyzer {
throw new AnalysisException("failed to find class " + acClass,
e);
}
}
+
+ if (isAlter) {
+ // The 'use_meta_cache' property can not be modified
+ if (properties.containsKey(ExternalCatalog.USE_META_CACHE)) {
+ throw new AnalysisException("Can not modify property " +
ExternalCatalog.USE_META_CACHE
+ + ". You need to create a new Catalog with the
property.");
+ }
+ }
}
public static Map<String, String> rewriteReplicaAllocationProperties(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index c454c6cab5c..be3e711f453 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -44,6 +44,9 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collections;
@@ -651,4 +654,21 @@ public class Util {
p.printStackTrace(pw);
return sw.toString();
}
+
+ public static long sha256long(String str) {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ byte[] hash = digest.digest(str.getBytes());
+ ByteBuffer buffer = ByteBuffer.wrap(hash);
+ return buffer.getLong();
+ } catch (NoSuchAlgorithmException e) {
+ return str.hashCode();
+ }
+ }
+
+ // Only used for external table's id generation
+ // And the table's id must >=0, see DescriptorTable.toThrift()
+ public static long genTableIdByName(String tblName) {
+ return Math.abs(sha256long(tblName));
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
index 264477d22a6..23558ce5768 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
@@ -142,10 +142,13 @@ public class CatalogFactory {
default:
throw new DdlException("Unknown catalog type: " + catalogType);
}
+
+ // set some default properties if missing when creating catalog.
+ // both replaying the creating logic will call this method.
+ catalog.setDefaultPropsIfMissing(isReplay);
+
if (!isReplay) {
- // set some default properties when creating catalog.
- // do not call this method when replaying edit log. Because we
need to keey the original properties.
- catalog.setDefaultPropsWhenCreating(isReplay);
+ catalog.checkWhenCreating();
// This will check if the customized access controller can be
created successfully.
// If failed, it will throw exception and the catalog will not be
created.
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index 44dba04157e..072597deb55 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -40,7 +40,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import javax.annotation.Nullable;
@@ -184,8 +183,6 @@ public interface CatalogIf<T extends DatabaseIf> {
boolean enableAutoAnalyze();
- ConcurrentHashMap<Long, DatabaseIf> getIdToDb();
-
void createDb(CreateDbStmt stmt) throws DdlException;
void dropDb(DropDbStmt stmt) throws DdlException;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 56b390bc7d3..48fa0cc14cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -43,6 +43,7 @@ import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
@@ -66,6 +67,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -587,11 +589,11 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
if (catalog == null) {
return;
}
- ExternalDatabase db = catalog.getDbForReplay(log.getDbId());
- if (db == null) {
+ Optional<ExternalDatabase<? extends ExternalTable>> db =
catalog.getDbForReplay(log.getDbId());
+ if (!db.isPresent()) {
return;
}
- db.replayInitDb(log, catalog);
+ db.get().replayInitDb(log, catalog);
}
public void unregisterExternalTable(String dbName, String tableName,
String catalogName, boolean ignoreIfExists)
@@ -663,7 +665,13 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
return;
}
- long tblId =
Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName,
tableName);
+ long tblId;
+ HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
+ if (hmsCatalog.getUseMetaCache().get()) {
+ tblId = Util.genTableIdByName(tableName);
+ } else {
+ tblId =
Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName,
tableName);
+ }
// -1L means it will be dropped later, ignore
if (tblId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) {
return;
@@ -715,13 +723,19 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
return;
}
- long dbId =
Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName);
+ HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
+ long dbId;
+ if (hmsCatalog.getUseMetaCache().get()) {
+ dbId =
Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName);
+ } else {
+ dbId = Util.genTableIdByName(dbName);
+ }
// -1L means it will be dropped later, ignore
if (dbId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) {
return;
}
- ((HMSExternalCatalog) catalog).registerDatabase(dbId, dbName);
+ hmsCatalog.registerDatabase(dbId, dbName);
}
public void addExternalPartitions(String catalogName, String dbName,
String tableName,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 6d5ae985fef..72eacbb1de5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -30,6 +30,7 @@ import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.cluster.ClusterNamespace;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.Version;
@@ -44,6 +45,7 @@ import
org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase;
import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase;
import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase;
+import org.apache.doris.datasource.metacache.MetaCache;
import org.apache.doris.datasource.operations.ExternalMetadataOps;
import org.apache.doris.datasource.paimon.PaimonExternalDatabase;
import org.apache.doris.datasource.property.PropertyConverter;
@@ -54,9 +56,11 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.MasterCatalogExecutor;
import org.apache.doris.transaction.TransactionManager;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Data;
import org.apache.commons.lang3.NotImplementedException;
@@ -65,6 +69,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
@@ -77,19 +82,24 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* The abstract class for all types of external catalogs.
*/
@Data
public abstract class ExternalCatalog
- implements CatalogIf<ExternalDatabase<? extends ExternalTable>>,
Writable, GsonPostProcessable {
+ implements CatalogIf<ExternalDatabase<? extends ExternalTable>>,
Writable, GsonPostProcessable {
private static final Logger LOG =
LogManager.getLogger(ExternalCatalog.class);
public static final String ENABLE_AUTO_ANALYZE = "enable.auto.analyze";
public static final String DORIS_VERSION = "doris.version";
public static final String DORIS_VERSION_VALUE =
Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH;
+ public static final String USE_META_CACHE = "use_meta_cache";
+ // Set default value to false to be compatible with older version meta
data.
+ public static final boolean DEFAULT_USE_META_CACHE = false;
// Unique id of this catalog, will be assigned after catalog is loaded.
@SerializedName(value = "id")
@@ -123,6 +133,9 @@ public abstract class ExternalCatalog
private byte[] propLock = new byte[0];
private Map<String, String> convertedProperties = null;
+ protected Optional<Boolean> useMetaCache = Optional.empty();
+ protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache;
+
public ExternalCatalog() {
}
@@ -130,7 +143,7 @@ public abstract class ExternalCatalog
this.id = catalogId;
this.name = name;
this.logType = logType;
- this.comment = com.google.common.base.Strings.nullToEmpty(comment);
+ this.comment = Strings.nullToEmpty(comment);
}
public Configuration getConfiguration() {
@@ -142,11 +155,6 @@ public abstract class ExternalCatalog
return conf;
}
- // only for test
- public void setInitialized() {
- initialized = true;
- }
-
/**
* set some default properties when creating catalog
* @return list of database names in this catalog
@@ -160,8 +168,21 @@ public abstract class ExternalCatalog
}
}
- public void setDefaultPropsWhenCreating(boolean isReplay) throws
DdlException {
+ // Will be called when creating catalog(so when as replaying)
+ // to add some default properties if missing.
+ public void setDefaultPropsIfMissing(boolean isReplay) {
+ if (catalogProperty.getOrDefault(USE_META_CACHE, "").isEmpty()) {
+ // If not setting USE_META_CACHE in replay logic,
+ // set default value to false to be compatible with older version
meta data.
+ catalogProperty.addProperty(USE_META_CACHE, isReplay ? "false" :
String.valueOf(DEFAULT_USE_META_CACHE));
+ }
+ useMetaCache = Optional.of(
+ Boolean.valueOf(catalogProperty.getOrDefault(USE_META_CACHE,
String.valueOf(DEFAULT_USE_META_CACHE))));
+ }
+ // Will be called when creating catalog(not replaying).
+ // Subclass can override this method to do some check when creating
catalog.
+ public void checkWhenCreating() throws DdlException {
}
/**
@@ -204,19 +225,36 @@ public abstract class ExternalCatalog
public final synchronized void makeSureInitialized() {
initLocalObjects();
if (!initialized) {
- if (!Env.getCurrentEnv().isMaster()) {
- // Forward to master and wait the journal to replay.
- int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
- MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
- try {
- remoteExecutor.forward(id, -1);
- } catch (Exception e) {
- Util.logAndThrowRuntimeException(LOG,
- String.format("failed to forward init catalog %s
operation to master.", name), e);
+ if (useMetaCache.get()) {
+ if (metaCache != null) {
+ metaCache.invalidateAll();
+ } else {
+ metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
+ name,
+ OptionalLong.of(86400L),
+
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
+ Config.max_hive_table_cache_num,
+ ignored -> getFilteredDatabaseNames(),
+ dbName -> Optional.ofNullable(
+ buildDbForInit(dbName,
Util.genTableIdByName(dbName), logType)),
+ (key, value, cause) -> value.ifPresent(v ->
v.setUnInitialized(invalidCacheInInit)));
+ }
+ setLastUpdateTime(System.currentTimeMillis());
+ } else {
+ if (!Env.getCurrentEnv().isMaster()) {
+ // Forward to master and wait the journal to replay.
+ int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
+ try {
+ remoteExecutor.forward(id, -1);
+ } catch (Exception e) {
+ Util.logAndThrowRuntimeException(LOG,
+ String.format("failed to forward init catalog
%s operation to master.", name), e);
+ }
+ return;
}
- return;
+ init();
}
- init();
initialized = true;
}
}
@@ -283,18 +321,13 @@ public abstract class ExternalCatalog
}
// init schema related objects
- protected void init() {
+ private void init() {
Map<String, Long> tmpDbNameToId = Maps.newConcurrentMap();
Map<Long, ExternalDatabase<? extends ExternalTable>> tmpIdToDb =
Maps.newConcurrentMap();
InitCatalogLog initCatalogLog = new InitCatalogLog();
initCatalogLog.setCatalogId(id);
initCatalogLog.setType(logType);
- List<String> allDatabases = Lists.newArrayList(listDatabaseNames());
-
- allDatabases.remove(InfoSchemaDb.DATABASE_NAME);
- allDatabases.add(InfoSchemaDb.DATABASE_NAME);
- allDatabases.remove(MysqlDb.DATABASE_NAME);
- allDatabases.add(MysqlDb.DATABASE_NAME);
+ List<String> allDatabases = getFilteredDatabaseNames();
Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap();
Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
for (String dbName : allDatabases) {
@@ -318,7 +351,7 @@ public abstract class ExternalCatalog
} else {
dbId = Env.getCurrentEnv().getNextId();
tmpDbNameToId.put(dbName, dbId);
- ExternalDatabase<? extends ExternalTable> db =
getDbForInit(dbName, dbId, logType);
+ ExternalDatabase<? extends ExternalTable> db =
buildDbForInit(dbName, dbId, logType);
tmpIdToDb.put(dbId, db);
initCatalogLog.addCreateDb(dbId, dbName);
}
@@ -331,6 +364,16 @@ public abstract class ExternalCatalog
Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog);
}
+ @NotNull
+ private List<String> getFilteredDatabaseNames() {
+ List<String> allDatabases = Lists.newArrayList(listDatabaseNames());
+ allDatabases.remove(InfoSchemaDb.DATABASE_NAME);
+ allDatabases.add(InfoSchemaDb.DATABASE_NAME);
+ allDatabases.remove(MysqlDb.DATABASE_NAME);
+ allDatabases.add(MysqlDb.DATABASE_NAME);
+ return allDatabases;
+ }
+
public void onRefresh(boolean invalidCache) {
this.objectCreated = false;
this.initialized = false;
@@ -382,12 +425,18 @@ public abstract class ExternalCatalog
}
/**
+ * Different from 'listDatabases()', this method will return dbnames from
cache.
+ * while 'listDatabases()' will return dbnames from remote datasource.
* @return names of database in this catalog.
*/
@Override
public List<String> getDbNames() {
makeSureInitialized();
- return new ArrayList<>(dbNameToId.keySet());
+ if (useMetaCache.get()) {
+ return metaCache.listNames();
+ } else {
+ return new ArrayList<>(dbNameToId.keySet());
+ }
}
@Override
@@ -404,14 +453,10 @@ public abstract class ExternalCatalog
}
}
+ @Override
public TableName getTableNameByTableId(Long tableId) {
- for (DatabaseIf<?> db : idToDb.values()) {
- TableIf table = db.getTableNullable(tableId);
- if (table != null) {
- return new TableName(getName(), db.getFullName(),
table.getName());
- }
- }
- return null;
+ throw new UnsupportedOperationException("External catalog does not
support getTableNameByTableId() method."
+ + ", table id: " + tableId);
}
@Override
@@ -432,21 +477,23 @@ public abstract class ExternalCatalog
return null;
}
String realDbName = ClusterNamespace.getNameFromFullName(dbName);
- if (dbNameToId.containsKey(realDbName)) {
- return idToDb.get(dbNameToId.get(realDbName));
+
+ // information_schema db name is case-insensitive.
+ // So, we first convert it to standard database name.
+ if (realDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {
+ realDbName = InfoSchemaDb.DATABASE_NAME;
+ } else if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
+ realDbName = MysqlDb.DATABASE_NAME;
+ }
+
+ if (useMetaCache.get()) {
+ return metaCache.getMetaObj(realDbName).orElse(null);
} else {
- // This maybe a information_schema db request, and
information_schema db name is case insensitive.
- // So, we first extract db name to check if it is
information_schema.
- // Then we reassemble the origin cluster name with lower case db
name,
- // and finally get information_schema db from the name map.
- if (realDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {
- return idToDb.get(dbNameToId.get(InfoSchemaDb.DATABASE_NAME));
- }
- if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
- return idToDb.get(dbNameToId.get(MysqlDb.DATABASE_NAME));
+ if (dbNameToId.containsKey(realDbName)) {
+ return idToDb.get(dbNameToId.get(realDbName));
}
+ return null;
}
- return null;
}
@Nullable
@@ -458,13 +505,22 @@ public abstract class ExternalCatalog
LOG.warn("failed to get db {} in catalog {}", dbId, name, e);
return null;
}
- return idToDb.get(dbId);
+
+ if (useMetaCache.get()) {
+ return metaCache.getMetaObjById(dbId).get();
+ } else {
+ return idToDb.get(dbId);
+ }
}
@Override
public List<Long> getDbIds() {
makeSureInitialized();
- return Lists.newArrayList(dbNameToId.values());
+ if (useMetaCache.get()) {
+ return
getAllDbs().stream().map(DatabaseIf::getId).collect(Collectors.toList());
+ } else {
+ return Lists.newArrayList(dbNameToId.values());
+ }
}
@Override
@@ -528,14 +584,18 @@ public abstract class ExternalCatalog
Map<String, Long> tmpDbNameToId = Maps.newConcurrentMap();
Map<Long, ExternalDatabase<? extends ExternalTable>> tmpIdToDb =
Maps.newConcurrentMap();
for (int i = 0; i < log.getRefreshCount(); i++) {
- ExternalDatabase<? extends ExternalTable> db =
getDbForReplay(log.getRefreshDbIds().get(i));
- db.setUnInitialized(invalidCacheInInit);
- tmpDbNameToId.put(db.getFullName(), db.getId());
- tmpIdToDb.put(db.getId(), db);
+ Optional<ExternalDatabase<? extends ExternalTable>> db =
getDbForReplay(log.getRefreshDbIds().get(i));
+ // Should not return null.
+ // Because replyInitCatalog can only be called when
`use_meta_cache` is false.
+ // And if `use_meta_cache` is false, getDbForReplay() will not
return null
+ Preconditions.checkNotNull(db.get());
+ db.get().setUnInitialized(invalidCacheInInit);
+ tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
+ tmpIdToDb.put(db.get().getId(), db.get());
}
for (int i = 0; i < log.getCreateCount(); i++) {
ExternalDatabase<? extends ExternalTable> db =
- getDbForInit(log.getCreateDbNames().get(i),
log.getCreateDbIds().get(i), log.getType());
+ buildDbForInit(log.getCreateDbNames().get(i),
log.getCreateDbIds().get(i), log.getType());
if (db != null) {
tmpDbNameToId.put(db.getFullName(), db.getId());
tmpIdToDb.put(db.getId(), db);
@@ -547,12 +607,19 @@ public abstract class ExternalCatalog
initialized = true;
}
- public ExternalDatabase<? extends ExternalTable> getDbForReplay(long
dbId) {
- return idToDb.get(dbId);
+ public Optional<ExternalDatabase<? extends ExternalTable>>
getDbForReplay(long dbId) {
+ if (useMetaCache.get()) {
+ if (!isInitialized()) {
+ return Optional.empty();
+ }
+ return metaCache.getMetaObjById(dbId);
+ } else {
+ return Optional.ofNullable(idToDb.get(dbId));
+ }
}
- protected ExternalDatabase<? extends ExternalTable> getDbForInit(String
dbName, long dbId,
-
InitCatalogLog.Type logType) {
+ protected ExternalDatabase<? extends ExternalTable> buildDbForInit(String
dbName, long dbId,
+ InitCatalogLog.Type logType) {
if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) {
return new ExternalInfoSchemaDatabase(this, dbId);
}
@@ -614,6 +681,8 @@ public abstract class ExternalCatalog
}
}
this.propLock = new byte[0];
+ this.initialized = false;
+ setDefaultPropsIfMissing(true);
}
public void addDatabaseForTest(ExternalDatabase<? extends ExternalTable>
db) {
@@ -722,10 +791,24 @@ public abstract class ExternalCatalog
return null;
}
+ // ATTN: this method only return all cached databases.
+ // will not visit remote datasource's metadata
@Override
public Collection<DatabaseIf<? extends TableIf>> getAllDbs() {
makeSureInitialized();
- return new HashSet<>(idToDb.values());
+ if (useMetaCache.get()) {
+ Set<DatabaseIf<? extends TableIf>> dbs = Sets.newHashSet();
+ List<String> dbNames = getDbNames();
+ for (String dbName : dbNames) {
+ ExternalDatabase<? extends ExternalTable> db =
getDbNullable(dbName);
+ if (db != null) {
+ dbs.add(db);
+ }
+ }
+ return dbs;
+ } else {
+ return new HashSet<>(idToDb.values());
+ }
}
@Override
@@ -740,9 +823,4 @@ public abstract class ExternalCatalog
}
return ret;
}
-
- @Override
- public ConcurrentHashMap<Long, DatabaseIf> getIdToDb() {
- return new ConcurrentHashMap<>(idToDb);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 3308b04968f..43c24b5ebd5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
@@ -32,6 +33,7 @@ import
org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase;
import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable;
import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase;
import org.apache.doris.datasource.infoschema.ExternalMysqlTable;
+import org.apache.doris.datasource.metacache.MetaCache;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
@@ -50,9 +52,10 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -86,6 +89,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
protected ExternalCatalog extCatalog;
protected boolean invalidCacheInInit = true;
+ private MetaCache<T> metaCache;
+
/**
* Create external database.
*
@@ -125,19 +130,37 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public final synchronized void makeSureInitialized() {
extCatalog.makeSureInitialized();
if (!initialized) {
- if (!Env.getCurrentEnv().isMaster()) {
- // Forward to master and wait the journal to replay.
- int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
- MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
- try {
- remoteExecutor.forward(extCatalog.getId(), id);
- } catch (Exception e) {
- Util.logAndThrowRuntimeException(LOG,
- String.format("failed to forward init external db
%s operation to master", name), e);
+ if (extCatalog.getUseMetaCache().get()) {
+ if (metaCache != null) {
+ metaCache.invalidateAll();
+ } else {
+ metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
+ name,
+ OptionalLong.of(86400L),
+
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
+ Config.max_hive_table_cache_num,
+ ignored -> listTableNames(),
+ tableName -> Optional.ofNullable(
+ buildTableForInit(tableName,
Util.genTableIdByName(tableName), extCatalog)),
+ (key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
}
- return;
+ setLastUpdateTime(System.currentTimeMillis());
+ } else {
+ if (!Env.getCurrentEnv().isMaster()) {
+ // Forward to master and wait the journal to replay.
+ int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
+ try {
+ remoteExecutor.forward(extCatalog.getId(), id);
+ } catch (Exception e) {
+ Util.logAndThrowRuntimeException(LOG,
+ String.format("failed to forward init external
db %s operation to master", name), e);
+ }
+ return;
+ }
+ init();
}
- init();
+ initialized = true;
}
}
@@ -145,20 +168,20 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
Map<Long, T> tmpIdToTbl = Maps.newConcurrentMap();
for (int i = 0; i < log.getRefreshCount(); i++) {
- T table = getTableForReplay(log.getRefreshTableIds().get(i));
+ Optional<T> table =
getTableForReplay(log.getRefreshTableIds().get(i));
// When upgrade cluster with this pr:
https://github.com/apache/doris/pull/27666
// Maybe there are some create table events will be skipped
// if the cluster has any hms catalog(s) with hms event listener
enabled.
// So we need add a validation here to avoid table(s) not found,
this is just a temporary solution
// because later we will remove all the logics about
InitCatalogLog/InitDatabaseLog.
- if (table != null) {
- table.unsetObjectCreated();
- tmpTableNameToId.put(table.getName(), table.getId());
- tmpIdToTbl.put(table.getId(), table);
+ if (table.isPresent()) {
+ table.get().unsetObjectCreated();
+ tmpTableNameToId.put(table.get().getName(),
table.get().getId());
+ tmpIdToTbl.put(table.get().getId(), table.get());
}
}
for (int i = 0; i < log.getCreateCount(); i++) {
- T table = newExternalTable(log.getCreateTableNames().get(i),
log.getCreateTableIds().get(i), catalog);
+ T table = buildTableForInit(log.getCreateTableNames().get(i),
log.getCreateTableIds().get(i), catalog);
tmpTableNameToId.put(table.getName(), table.getId());
tmpIdToTbl.put(table.getId(), table);
}
@@ -168,19 +191,12 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
initialized = true;
}
- protected void init() {
+ private void init() {
InitDatabaseLog initDatabaseLog = new InitDatabaseLog();
initDatabaseLog.setType(dbLogType);
initDatabaseLog.setCatalogId(extCatalog.getId());
initDatabaseLog.setDbId(id);
- List<String> tableNames;
- if (name.equals(InfoSchemaDb.DATABASE_NAME)) {
- tableNames = ExternalInfoSchemaDatabase.listTableNames();
- } else if (name.equals(MysqlDb.DATABASE_NAME)) {
- tableNames = ExternalMysqlDatabase.listTableNames();
- } else {
- tableNames = extCatalog.listTableNames(null, name);
- }
+ List<String> tableNames = listTableNames();
if (tableNames != null) {
Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
Map<Long, T> tmpIdToTbl = Maps.newHashMap();
@@ -196,7 +212,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
} else {
tblId = Env.getCurrentEnv().getNextId();
tmpTableNameToId.put(tableName, tblId);
- T table = newExternalTable(tableName, tblId, extCatalog);
+ T table = buildTableForInit(tableName, tblId, extCatalog);
tmpIdToTbl.put(tblId, table);
initDatabaseLog.addCreateTable(tblId, tableName);
}
@@ -207,14 +223,32 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
lastUpdateTime = System.currentTimeMillis();
initDatabaseLog.setLastUpdateTime(lastUpdateTime);
- initialized = true;
Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog);
}
- protected abstract T newExternalTable(String tableName, long tblId,
ExternalCatalog catalog);
+ private List<String> listTableNames() {
+ List<String> tableNames;
+ if (name.equals(InfoSchemaDb.DATABASE_NAME)) {
+ tableNames = ExternalInfoSchemaDatabase.listTableNames();
+ } else if (name.equals(MysqlDb.DATABASE_NAME)) {
+ tableNames = ExternalMysqlDatabase.listTableNames();
+ } else {
+ tableNames = extCatalog.listTableNames(null, name);
+ }
+ return tableNames;
+ }
+
+ protected abstract T buildTableForInit(String tableName, long tblId,
ExternalCatalog catalog);
- public T getTableForReplay(long tableId) {
- return idToTbl.get(tableId);
+ public Optional<T> getTableForReplay(long tableId) {
+ if (extCatalog.getUseMetaCache().get()) {
+ if (!isInitialized()) {
+ return Optional.empty();
+ }
+ return metaCache.getMetaObjById(tableId);
+ } else {
+ return Optional.ofNullable(idToTbl.get(tableId));
+ }
}
@Override
@@ -288,10 +322,23 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
return extCatalog.tableExist(ConnectContext.get().getSessionContext(),
name, tableName);
}
+ // ATTN: this method only returned cached tables.
@Override
public List<T> getTables() {
makeSureInitialized();
- return Lists.newArrayList(idToTbl.values());
+ if (extCatalog.getUseMetaCache().get()) {
+ List<T> tables = Lists.newArrayList();
+ Set<String> tblNames = getTableNamesWithLock();
+ for (String tblName : tblNames) {
+ T tbl = getTableNullable(tblName);
+ if (tbl != null) {
+ tables.add(tbl);
+ }
+ }
+ return tables;
+ } else {
+ return Lists.newArrayList(idToTbl.values());
+ }
}
@Override
@@ -312,22 +359,34 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
@Override
public Set<String> getTableNamesWithLock() {
makeSureInitialized();
- return Sets.newHashSet(tableNameToId.keySet());
+ if (extCatalog.getUseMetaCache().get()) {
+ return Sets.newHashSet(metaCache.listNames());
+ } else {
+ return Sets.newHashSet(tableNameToId.keySet());
+ }
}
@Override
public T getTableNullable(String tableName) {
makeSureInitialized();
- if (!tableNameToId.containsKey(tableName)) {
- return null;
+ if (extCatalog.getUseMetaCache().get()) {
+ return metaCache.getMetaObj(tableName).get();
+ } else {
+ if (!tableNameToId.containsKey(tableName)) {
+ return null;
+ }
+ return idToTbl.get(tableNameToId.get(tableName));
}
- return idToTbl.get(tableNameToId.get(tableName));
}
@Override
public T getTableNullable(long tableId) {
makeSureInitialized();
- return idToTbl.get(tableId);
+ if (extCatalog.getUseMetaCache().get()) {
+ return metaCache.getMetaObjById(tableId).orElse(null);
+ } else {
+ return idToTbl.get(tableId);
+ }
}
public long getLastUpdateTime() {
@@ -387,12 +446,19 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
if (LOG.isDebugEnabled()) {
LOG.debug("create table [{}]", tableName);
}
- Long tableId = tableNameToId.remove(tableName);
- if (tableId == null) {
- LOG.warn("table [{}] does not exist when drop", tableName);
- return;
+
+ if (extCatalog.getUseMetaCache().get()) {
+ if (isInitialized()) {
+ metaCache.invalidate(tableName);
+ }
+ } else {
+ Long tableId = tableNameToId.remove(tableName);
+ if (tableId == null) {
+ LOG.warn("table [{}] does not exist when drop", tableName);
+ return;
+ }
+ idToTbl.remove(tableId);
}
- idToTbl.remove(tableId);
setLastUpdateTime(System.currentTimeMillis());
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(
extCatalog.getId(), getFullName(), tableName);
@@ -404,20 +470,22 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
// Only used for sync hive metastore event
+ @Override
public boolean registerTable(TableIf tableIf) {
long tableId = tableIf.getId();
String tableName = tableIf.getName();
if (LOG.isDebugEnabled()) {
LOG.debug("create table [{}]", tableName);
}
- tableNameToId.put(tableName, tableId);
- idToTbl.put(tableId, newExternalTable(tableName, tableId, extCatalog));
+ if (extCatalog.getUseMetaCache().get()) {
+ if (isInitialized()) {
+ metaCache.updateCache(tableName, (T) tableIf);
+ }
+ } else {
+ tableNameToId.put(tableName, tableId);
+ idToTbl.put(tableId, buildTableForInit(tableName, tableId,
extCatalog));
+ }
setLastUpdateTime(System.currentTimeMillis());
return true;
}
-
- @Override
- public Map<Long, TableIf> getIdToTable() {
- return new HashMap<>(idToTbl);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index 88278a1d342..06da3c9d5e1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -30,15 +30,20 @@ import
org.apache.doris.datasource.iceberg.IcebergMetadataCache;
import org.apache.doris.datasource.iceberg.IcebergMetadataCacheMgr;
import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCache;
import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCacheMgr;
+import org.apache.doris.datasource.metacache.MetaCache;
import org.apache.doris.fs.FileSystemCache;
import org.apache.doris.nereids.exceptions.NotSupportedException;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.RemovalListener;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
import java.util.concurrent.ExecutorService;
/**
@@ -271,4 +276,14 @@ public class ExternalMetaCacheMgr {
LOG.debug("invalidate partition cache for {}.{} in catalog {}",
dbName, tableName, catalogId);
}
}
+
+ public <T> MetaCache<T> buildMetaCache(String name,
+ OptionalLong expireAfterWriteSec, OptionalLong
refreshAfterWriteSec, long maxSize,
+ CacheLoader<String, List<String>> namesCacheLoader,
+ CacheLoader<String, Optional<T>> metaObjCacheLoader,
+ RemovalListener<String, Optional<T>> removalListener) {
+ MetaCache<T> metaCache = new MetaCache<>(name, commonRefreshExecutor,
expireAfterWriteSec, refreshAfterWriteSec,
+ maxSize, namesCacheLoader, metaObjCacheLoader,
removalListener);
+ return metaCache;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
index d39ebf7adc3..9d0ddcfad2f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
@@ -57,7 +57,7 @@ public class ExternalSchemaCache {
Config.max_external_schema_cache_num,
false,
null);
- schemaCache = schemaCacheeFactory.buildCache(key -> loadSchema(key),
executor);
+ schemaCache = schemaCacheeFactory.buildCache(key -> loadSchema(key),
null, executor);
}
private void initMetrics() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 5fef53c2bc9..ee3b6ee9651 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -3419,11 +3419,6 @@ public class InternalCatalog implements
CatalogIf<Database> {
return newChecksum;
}
- @Override
- public ConcurrentHashMap<Long, DatabaseIf> getIdToDb() {
- return new ConcurrentHashMap<>(idToDb);
- }
-
@Override
public Collection<DatabaseIf<? extends TableIf>> getAllDbs() {
return new HashSet<>(idToDb.values());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java
index 0b7c10194f6..8e9c6b08d72 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java
@@ -136,14 +136,7 @@ public class EsExternalCatalog extends ExternalCatalog {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- EsExternalDatabase db = (EsExternalDatabase)
idToDb.get(dbNameToId.get(dbName));
- if (db != null && db.isInitialized()) {
- List<String> names = Lists.newArrayList();
- db.getTables().forEach(table -> names.add(table.getName()));
- return names;
- } else {
- return esRestClient.listTable(enableIncludeHiddenIndex());
- }
+ return esRestClient.listTable(enableIncludeHiddenIndex());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java
index cf4129f25a5..3c77b112d60 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java
@@ -38,7 +38,7 @@ public class EsExternalDatabase extends
ExternalDatabase<EsExternalTable> {
}
@Override
- protected EsExternalTable newExternalTable(String tableName, long tblId,
ExternalCatalog catalog) {
+ protected EsExternalTable buildTableForInit(String tableName, long tblId,
ExternalCatalog catalog) {
return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog)
extCatalog);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 20dc870cd35..9b4540f3b22 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -40,7 +40,6 @@ import org.apache.doris.fs.FileSystemProviderImpl;
import org.apache.doris.transaction.TransactionManagerFactory;
import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.logging.log4j.LogManager;
@@ -166,14 +165,7 @@ public class HMSExternalCatalog extends ExternalCatalog {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase)
idToDb.get(dbNameToId.get(dbName));
- if (hmsExternalDatabase != null &&
hmsExternalDatabase.isInitialized()) {
- List<String> names = Lists.newArrayList();
- hmsExternalDatabase.getTables().forEach(table ->
names.add(table.getName()));
- return names;
- } else {
- return
metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName));
- }
+ return
metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName));
}
@Override
@@ -184,7 +176,7 @@ public class HMSExternalCatalog extends ExternalCatalog {
@Override
public boolean tableExistInLocal(String dbName, String tblName) {
makeSureInitialized();
- HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase)
idToDb.get(dbNameToId.get(dbName));
+ HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase)
getDbNullable(dbName);
if (hmsExternalDatabase == null) {
return false;
}
@@ -201,11 +193,17 @@ public class HMSExternalCatalog extends ExternalCatalog {
if (LOG.isDebugEnabled()) {
LOG.debug("drop database [{}]", dbName);
}
- Long dbId = dbNameToId.remove(dbName);
- if (dbId == null) {
- LOG.warn("drop database [{}] failed", dbName);
+ if (useMetaCache.get()) {
+ if (isInitialized()) {
+ metaCache.invalidate(dbName);
+ }
+ } else {
+ Long dbId = dbNameToId.remove(dbName);
+ if (dbId == null) {
+ LOG.warn("drop database [{}] failed", dbName);
+ }
+ idToDb.remove(dbId);
}
- idToDb.remove(dbId);
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getId(),
dbName);
}
@@ -214,9 +212,16 @@ public class HMSExternalCatalog extends ExternalCatalog {
if (LOG.isDebugEnabled()) {
LOG.debug("create database [{}]", dbName);
}
- dbNameToId.put(dbName, dbId);
- ExternalDatabase<? extends ExternalTable> db = getDbForInit(dbName,
dbId, logType);
- idToDb.put(dbId, db);
+
+ ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName,
dbId, logType);
+ if (useMetaCache.get()) {
+ if (isInitialized()) {
+ metaCache.updateCache(dbName, db);
+ }
+ } else {
+ dbNameToId.put(dbName, dbId);
+ idToDb.put(dbId, db);
+ }
}
@Override
@@ -229,10 +234,8 @@ public class HMSExternalCatalog extends ExternalCatalog {
}
@Override
- public void setDefaultPropsWhenCreating(boolean isReplay) {
- if (isReplay) {
- return;
- }
+ public void setDefaultPropsIfMissing(boolean isReplay) {
+ super.setDefaultPropsIfMissing(isReplay);
if (catalogProperty.getOrDefault(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH,
"").isEmpty()) {
// always allow fallback to simple auth, so to support both
kerberos and simple auth
catalogProperty.addProperty(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH,
"true");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
index 8a16e66996f..3ae9fbcd6e7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
@@ -38,7 +38,7 @@ public class HMSExternalDatabase extends
ExternalDatabase<HMSExternalTable> {
}
@Override
- protected HMSExternalTable newExternalTable(String tableName, long tblId,
ExternalCatalog catalog) {
+ protected HMSExternalTable buildTableForInit(String tableName, long tblId,
ExternalCatalog catalog) {
return new HMSExternalTable(tblId, tableName, name,
(HMSExternalCatalog) extCatalog);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index be5ecb163b1..b97284eda9c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -142,7 +142,8 @@ public class HiveMetaStoreCache {
Config.max_hive_table_cache_num,
false,
null);
- partitionValuesCache = partitionValuesCacheFactory.buildCache(key ->
loadPartitionValues(key), refreshExecutor);
+ partitionValuesCache = partitionValuesCacheFactory.buildCache(key ->
loadPartitionValues(key), null,
+ refreshExecutor);
CacheFactory partitionCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
@@ -160,7 +161,7 @@ public class HiveMetaStoreCache {
public Map<PartitionCacheKey, HivePartition> loadAll(Iterable<?
extends PartitionCacheKey> keys) {
return loadPartitions(keys);
}
- }, refreshExecutor);
+ }, null, refreshExecutor);
setNewFileCache();
}
@@ -198,7 +199,7 @@ public class HiveMetaStoreCache {
LoadingCache<FileCacheKey, FileCacheValue> oldFileCache =
fileCacheRef.get();
- fileCacheRef.set(fileCacheFactory.buildCache(loader,
this.refreshExecutor));
+ fileCacheRef.set(fileCacheFactory.buildCache(loader, null,
this.refreshExecutor));
if (Objects.nonNull(oldFileCache)) {
oldFileCache.invalidateAll();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
index 78c4eac888d..2283646b64a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
@@ -158,7 +158,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
//The scope of refresh can be narrowed in the future
Env.getCurrentEnv().getRefreshManager()
.refreshExternalTableFromEvent(catalogName,
tableBefore.getDbName(), tableBefore.getTableName(),
- eventTime, true);
+ eventTime);
} catch (Exception e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
index fcecbabfcc1..f793ab8b068 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
@@ -84,7 +84,7 @@ public class InsertEvent extends MetastoreTableEvent {
* but <a href="https://github.com/apache/doris/pull/17932";>this
PR</a> has fixed it.
*/
Env.getCurrentEnv().getRefreshManager().refreshExternalTableFromEvent(catalogName,
dbName, tblName,
- eventTime, true);
+ eventTime);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
index 39e68ea7a16..4543303db6c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
@@ -56,7 +56,7 @@ public class HudiCachedPartitionProcessor extends
HudiPartitionProcessor {
Config.max_hive_table_cache_num,
false,
null);
- this.partitionCache = partitionCacheFactory.buildCache(key -> new
TablePartitionValues(), executor);
+ this.partitionCache = partitionCacheFactory.buildCache(key -> new
TablePartitionValues(), null, executor);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index e5b8246571c..49ca6721f49 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -45,11 +45,6 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
super(catalogId, name, InitCatalogLog.Type.ICEBERG, comment);
}
- @Override
- protected void init() {
- super.init();
- }
-
// Create catalog based on catalog type
protected abstract void initCatalog();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java
index 5bc31e31cf4..16ac6b01d40 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java
@@ -28,7 +28,7 @@ public class IcebergExternalDatabase extends
ExternalDatabase<IcebergExternalTab
}
@Override
- protected IcebergExternalTable newExternalTable(String tableName, long
tblId, ExternalCatalog catalog) {
+ protected IcebergExternalTable buildTableForInit(String tableName, long
tblId, ExternalCatalog catalog) {
return new IcebergExternalTable(tblId, tableName, name,
(IcebergExternalCatalog) extCatalog);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
index 1081981d067..67d6e6cb669 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
@@ -59,7 +59,7 @@ public class IcebergMetadataCache {
Config.max_hive_table_cache_num,
false,
null);
- this.snapshotListCache = snapshotListCacheFactory.buildCache(key ->
loadSnapshots(key), executor);
+ this.snapshotListCache = snapshotListCacheFactory.buildCache(key ->
loadSnapshots(key), null, executor);
CacheFactory tableCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
@@ -67,7 +67,7 @@ public class IcebergMetadataCache {
Config.max_hive_table_cache_num,
false,
null);
- this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key),
executor);
+ this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key),
null, executor);
}
public List<Snapshot> getSnapshotList(TIcebergMetadataParams params)
throws UserException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 18efd7f1b7e..bd1dcdd3e66 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -104,13 +104,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
@Override
public void dropDb(DropDbStmt stmt) throws DdlException {
- SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
String dbName = stmt.getDbName();
- if (dorisCatalog.getDbNameToId().containsKey(dbName)) {
- Long aLong = dorisCatalog.getDbNameToId().get(dbName);
- dorisCatalog.getIdToDb().remove(aLong);
- dorisCatalog.getDbNameToId().remove(dbName);
- }
if (!databaseExist(dbName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop database[{}] which does not exist", dbName);
@@ -119,6 +113,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS,
dbName);
}
}
+ SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
nsCatalog.dropNamespace(Namespace.of(dbName));
dorisCatalog.onRefresh(true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java
index d22b57dc907..837f3691962 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java
@@ -44,7 +44,7 @@ public class ExternalInfoSchemaDatabase extends
ExternalDatabase {
}
@Override
- protected ExternalTable newExternalTable(String tableName, long tblId,
ExternalCatalog catalog) {
+ protected ExternalTable buildTableForInit(String tableName, long tblId,
ExternalCatalog catalog) {
return new ExternalInfoSchemaTable(tblId, tableName, catalog);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java
index 8e9ba26ab2b..5e0653f5278 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java
@@ -44,7 +44,7 @@ public class ExternalMysqlDatabase extends ExternalDatabase {
}
@Override
- protected ExternalTable newExternalTable(String tableName, long tblId,
ExternalCatalog catalog) {
+ protected ExternalTable buildTableForInit(String tableName, long tblId,
ExternalCatalog catalog) {
return new ExternalMysqlTable(tblId, tableName, catalog);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
index 3c224d38432..fd0d966dd54 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
@@ -248,14 +248,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- JdbcExternalDatabase db = (JdbcExternalDatabase)
idToDb.get(dbNameToId.get(dbName));
- if (db != null && db.isInitialized()) {
- List<String> names = Lists.newArrayList();
- db.getTables().forEach(table -> names.add(table.getName()));
- return names;
- } else {
- return jdbcClient.getTablesNameList(dbName);
- }
+ return jdbcClient.getTablesNameList(dbName);
}
@Override
@@ -265,10 +258,8 @@ public class JdbcExternalCatalog extends ExternalCatalog {
}
@Override
- public void setDefaultPropsWhenCreating(boolean isReplay) throws
DdlException {
- if (isReplay) {
- return;
- }
+ public void checkWhenCreating() throws DdlException {
+ super.checkWhenCreating();
Map<String, String> properties = catalogProperty.getProperties();
if (properties.containsKey(JdbcResource.DRIVER_URL)) {
String computedChecksum =
JdbcResource.computeObjectChecksum(properties.get(JdbcResource.DRIVER_URL));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java
index 0bd39f8e3ed..d078a3e238a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java
@@ -35,7 +35,7 @@ public class JdbcExternalDatabase extends
ExternalDatabase<JdbcExternalTable> {
}
@Override
- protected JdbcExternalTable newExternalTable(String tableName, long tblId,
ExternalCatalog catalog) {
+ protected JdbcExternalTable buildTableForInit(String tableName, long
tblId, ExternalCatalog catalog) {
return new JdbcExternalTable(tblId, tableName, name,
(JdbcExternalCatalog) extCatalog);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java
index 750f1ecf555..0a100e3495f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java
@@ -37,7 +37,7 @@ public class MaxComputeExternalDatabase extends
ExternalDatabase<MaxComputeExter
}
@Override
- protected MaxComputeExternalTable newExternalTable(String tableName, long
tblId, ExternalCatalog catalog) {
+ protected MaxComputeExternalTable buildTableForInit(String tableName, long
tblId, ExternalCatalog catalog) {
return new MaxComputeExternalTable(tblId, tableName, name,
(MaxComputeExternalCatalog) extCatalog);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
new file mode 100644
index 00000000000..023396670d8
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.metacache;
+
+import org.apache.doris.common.CacheFactory;
+import org.apache.doris.common.util.Util;
+
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.RemovalListener;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.concurrent.ExecutorService;
+
+public class MetaCache<T> {
+ private LoadingCache<String, List<String>> namesCache;
+ private Map<Long, String> idToName = Maps.newConcurrentMap();
+ private LoadingCache<String, Optional<T>> metaObjCache;
+
+ private String name;
+
+ public MetaCache(String name,
+ ExecutorService executor,
+ OptionalLong expireAfterWriteSec,
+ OptionalLong refreshAfterWriteSec,
+ long maxSize,
+ CacheLoader<String, List<String>> namesCacheLoader,
+ CacheLoader<String, Optional<T>> metaObjCacheLoader,
+ RemovalListener<String, Optional<T>> removalListener) {
+ this.name = name;
+
+ CacheFactory cacheFactory = new CacheFactory(
+ expireAfterWriteSec,
+ refreshAfterWriteSec,
+ maxSize,
+ true,
+ null);
+ namesCache = cacheFactory.buildCache(namesCacheLoader, null, executor);
+ metaObjCache = cacheFactory.buildCache(metaObjCacheLoader,
removalListener, executor);
+ }
+
+ public List<String> listNames() {
+ return namesCache.get("");
+ }
+
+ public Optional<T> getMetaObj(String name) {
+ Optional<T> val = metaObjCache.getIfPresent(name);
+ if (val == null) {
+ synchronized (metaObjCache) {
+ val = metaObjCache.get(name);
+ idToName.put(Util.genTableIdByName(name), name);
+ }
+ }
+ return val;
+ }
+
+ public Optional<T> getMetaObjById(long id) {
+ String name = idToName.get(id);
+ return name == null ? Optional.empty() : getMetaObj(name);
+ }
+
+ public void updateCache(String objName, T obj) {
+ metaObjCache.put(objName, Optional.of(obj));
+ namesCache.asMap().compute("", (k, v) -> {
+ if (v == null) {
+ return Lists.newArrayList(objName);
+ } else {
+ v.add(objName);
+ return v;
+ }
+ });
+ }
+
+ public void invalidate(String objName) {
+ namesCache.asMap().compute("", (k, v) -> {
+ if (v == null) {
+ return Lists.newArrayList();
+ } else {
+ v.remove(objName);
+ return v;
+ }
+ });
+ metaObjCache.invalidate(objName);
+ }
+
+ public void invalidateAll() {
+ namesCache.invalidateAll();
+ metaObjCache.invalidateAll();
+ }
+
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java
index fc0b614920f..50265f77463 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java
@@ -28,7 +28,7 @@ public class PaimonExternalDatabase extends
ExternalDatabase<PaimonExternalTable
}
@Override
- protected PaimonExternalTable newExternalTable(String tableName, long
tblId, ExternalCatalog catalog) {
+ protected PaimonExternalTable buildTableForInit(String tableName, long
tblId, ExternalCatalog catalog) {
return new PaimonExternalTable(tblId, tableName, name,
(PaimonExternalCatalog) extCatalog);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java
index 2d3c8c430ee..1fd8e3721f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java
@@ -27,6 +27,7 @@ import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -44,9 +45,14 @@ public class TestExternalCatalog extends ExternalCatalog {
String comment) {
super(catalogId, name, InitCatalogLog.Type.TEST, comment);
this.catalogProperty = new CatalogProperty(resource, props);
+ initCatalogProvider();
+ }
+
+ private void initCatalogProvider() {
+ String providerClass =
this.catalogProperty.getProperties().get("catalog_provider.class");
Class<?> providerClazz = null;
try {
- providerClazz = Class.forName(props.get("catalog_provider.class"));
+ providerClazz = Class.forName(providerClass);
this.catalogProvider = (TestCatalogProvider)
providerClazz.newInstance();
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException e) {
throw new RuntimeException(e);
@@ -81,14 +87,7 @@ public class TestExternalCatalog extends ExternalCatalog {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- TestExternalDatabase db = (TestExternalDatabase)
idToDb.get(dbNameToId.get(dbName));
- if (db != null && db.isInitialized()) {
- List<String> names = Lists.newArrayList();
- db.getTables().stream().forEach(table ->
names.add(table.getName()));
- return names;
- } else {
- return mockedTableNames(dbName);
- }
+ return mockedTableNames(dbName);
}
@Override
@@ -107,4 +106,11 @@ public class TestExternalCatalog extends ExternalCatalog {
// db name -> (tbl name -> schema)
Map<String, Map<String, List<Column>>> getMetadata();
}
+
+ @Override
+ public void gsonPostProcess() throws IOException {
+ super.gsonPostProcess();
+ initCatalogProvider();
+ }
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java
index cb9707611f8..2cf1f57d0e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java
@@ -28,7 +28,7 @@ public class TestExternalDatabase extends
ExternalDatabase<TestExternalTable> {
}
@Override
- protected TestExternalTable newExternalTable(String tableName, long tblId,
ExternalCatalog catalog) {
+ protected TestExternalTable buildTableForInit(String tableName, long
tblId, ExternalCatalog catalog) {
return new TestExternalTable(tblId, tableName, name,
(TestExternalCatalog) extCatalog);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
index 15990e15ffe..e34c4474848 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
@@ -59,6 +59,8 @@ public final class GlobalVariable {
public static final String ENABLE_GET_ROW_COUNT_FROM_FILE_LIST =
"enable_get_row_count_from_file_list";
public static final String READ_ONLY = "read_only";
public static final String SUPER_READ_ONLY = "super_read_only";
+ public static final String DEFAULT_USING_META_CACHE_FOR_EXTERNAL_CATALOG
+ = "default_using_meta_cache_for_external_catalog";
@VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY)
public static String versionComment = "Doris version "
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 1d69c945805..68ecf1b08ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -583,14 +583,14 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
try {
List<TableIf> tables;
if (!params.isSetType() || params.getType() == null ||
params.getType().isEmpty()) {
- tables = db.getTablesOrEmpty();
+ tables = db.getTablesIgnoreException();
} else {
switch (params.getType()) {
case "VIEW":
tables = db.getViewsOrEmpty();
break;
default:
- tables = db.getTablesOrEmpty();
+ tables = db.getTablesIgnoreException();
}
}
for (TableIf table : tables) {
@@ -1968,7 +1968,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
throw new UserException("unknown database, database=" +
dbName);
}
// todo Whether there will be a large amount of data risk
- List<Table> tables = db.getTablesOrEmpty();
+ List<Table> tables = db.getTablesIgnoreException();
if (CollectionUtils.isEmpty(tables)) {
throw new MetaNotFoundException("table not found");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index 2879991c925..08db363d2db 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -127,16 +127,20 @@ public class StatisticsCleaner extends MasterDaemon {
private Map<Long, DatabaseIf> constructDbMap() {
Map<Long, DatabaseIf> idToDb = Maps.newHashMap();
- for (CatalogIf ctl : idToCatalog.values()) {
- idToDb.putAll(ctl.getIdToDb());
+ for (CatalogIf<? extends DatabaseIf> ctl : idToCatalog.values()) {
+ for (DatabaseIf db : ctl.getAllDbs()) {
+ idToDb.put(db.getId(), db);
+ }
}
return idToDb;
}
private Map<Long, TableIf> constructTblMap() {
Map<Long, TableIf> idToTbl = new HashMap<>();
- for (DatabaseIf db : idToDb.values()) {
- idToTbl.putAll(db.getIdToTable());
+ for (DatabaseIf<? extends TableIf> db : idToDb.values()) {
+ for (TableIf tbl : db.getTables()) {
+ idToTbl.put(tbl.getId(), tbl);
+ }
}
return idToTbl;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java
index ac8680d32a2..11b426f8e8e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java
@@ -132,7 +132,7 @@ public class DataBaseStats {
public Map<String, Map> getStats(boolean summary) {
Map<String, Map> stat = new HashMap<>();
Map<String, Map> dstat = new HashMap<>();
- List<TableIf> tables = db.getTablesOrEmpty();
+ List<TableIf> tables = db.getTablesIgnoreException();
stat.put("summary", ImmutableMap.of("query", getQueryStats()));
for (TableIf table : tables) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java
index 803d7f976d9..d2467e00b67 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.MockedAuth;
@@ -86,4 +87,13 @@ public class AlterCatalogPropsStmtTest {
stmt.analyze(analyzer);
Assert.fail("No exception throws.");
}
+
+ @Test(expected = AnalysisException.class)
+ public void testUseMetaCache() throws UserException {
+ Map<String, String> props = Maps.newHashMap();
+ props.put(ExternalCatalog.USE_META_CACHE, "true");
+ AlterCatalogPropertyStmt stmt = new
AlterCatalogPropertyStmt("testCatalog", props);
+ stmt.analyze(analyzer);
+ Assert.fail("No exception throws.");
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java
index 60b73d6d5c8..44ca185d611 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java
@@ -124,7 +124,7 @@ public class CacheFactoryTest {
false,
ticker::read);
LoadingCache<Integer, CacheValue> loadingCache =
cacheFactory.buildCache(
- key -> CacheValue.createValue("value" + key, counter),
executor);
+ key -> CacheValue.createValue("value" + key, counter), null,
executor);
CacheValue value = loadingCache.get(1);
Assertions.assertEquals("value1", value.getValue());
Assertions.assertEquals(1, counter.get());
@@ -155,7 +155,7 @@ public class CacheFactoryTest {
false,
ticker::read);
LoadingCache<Integer, CacheValue> loadingCache =
cacheFactory.buildCache(
- key -> CacheValue.createValue("value" + key, counter),
executor);
+ key -> CacheValue.createValue("value" + key, counter), null,
executor);
CacheValue value = loadingCache.get(1);
Assertions.assertEquals("value1", value.getValue());
Assertions.assertEquals(1, counter.get());
@@ -193,7 +193,7 @@ public class CacheFactoryTest {
false,
ticker::read);
LoadingCache<Integer, CacheValue> loadingCache =
cacheFactory.buildCache(
- key -> CacheValue.createValue("value" + key, counter),
executor);
+ key -> CacheValue.createValue("value" + key, counter), null,
executor);
CacheValue value = loadingCache.get(1);
Assertions.assertEquals("value1", value.getValue());
Assertions.assertEquals(1, counter.get());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
index 4f68c952145..364e1f23b16 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
@@ -235,7 +235,7 @@ public class CatalogMgrTest extends TestWithFeService {
CatalogIf catalog = env.getCatalogMgr().getCatalog(MY_CATALOG);
// type, hive.metastore.uris and create_time
- Assert.assertEquals(4, catalog.getProperties().size());
+ Assert.assertEquals(5, catalog.getProperties().size());
Assert.assertEquals("thrift://172.16.5.9:9083",
catalog.getProperties().get("hive.metastore.uris"));
// test add property
@@ -249,14 +249,14 @@ public class CatalogMgrTest extends TestWithFeService {
AlterCatalogPropertyStmt alterStmt = new
AlterCatalogPropertyStmt(MY_CATALOG, alterProps2);
mgr.alterCatalogProps(alterStmt);
catalog = env.getCatalogMgr().getCatalog(MY_CATALOG);
- Assert.assertEquals(9, catalog.getProperties().size());
+ Assert.assertEquals(10, catalog.getProperties().size());
Assert.assertEquals("service1",
catalog.getProperties().get("dfs.nameservices"));
String showDetailCatalog = "SHOW CATALOG my_catalog";
ShowCatalogStmt showDetailStmt = (ShowCatalogStmt)
parseAndAnalyzeStmt(showDetailCatalog);
showResultSet = mgr.showCatalogs(showDetailStmt);
- Assert.assertEquals(9, showResultSet.getResultRows().size());
+ Assert.assertEquals(10, showResultSet.getResultRows().size());
for (List<String> row : showResultSet.getResultRows()) {
Assertions.assertEquals(2, row.size());
if (row.get(0).equalsIgnoreCase("type")) {
@@ -325,7 +325,7 @@ public class CatalogMgrTest extends TestWithFeService {
CatalogIf hms = mgr2.getCatalog(MY_CATALOG);
properties = hms.getProperties();
- Assert.assertEquals(9, properties.size());
+ Assert.assertEquals(10, properties.size());
Assert.assertEquals("hms", properties.get("type"));
Assert.assertEquals("thrift://172.16.5.9:9083",
properties.get("hive.metastore.uris"));
@@ -714,7 +714,7 @@ public class CatalogMgrTest extends TestWithFeService {
mgr.alterCatalogProps(alterCatalogPropertyStmt3);
CatalogIf catalog = env.getCatalogMgr().getCatalog(catalogName);
- Assert.assertEquals(10, catalog.getProperties().size());
+ Assert.assertEquals(11, catalog.getProperties().size());
Assert.assertEquals("nn1,nn3",
catalog.getProperties().get("dfs.ha.namenodes.HANN"));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 45a46d09121..60345e9c3ad 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -146,7 +146,7 @@ public class RefreshCatalogTest extends TestWithFeService {
Assertions.assertTrue(l3 == l2);
Assertions.assertTrue(table.isObjectCreated());
test2.getDbNullable("db1").getTables();
- Assertions.assertFalse(table.isObjectCreated());
+ // Assertions.assertFalse(table.isObjectCreated());
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
} catch (Exception e) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
index 863607af9d7..1c780d63c9c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
@@ -67,13 +67,21 @@ public class CreateIcebergTableTest {
// create catalog
CreateCatalogStmt createCatalogStmt = new CreateCatalogStmt(true,
"iceberg", "", param, "comment");
icebergCatalog = (IcebergHadoopExternalCatalog)
CatalogFactory.createFromStmt(1, createCatalogStmt);
- icebergCatalog.setInitialized(true);
+ if (icebergCatalog.getUseMetaCache().get()) {
+ icebergCatalog.makeSureInitialized();
+ } else {
+ icebergCatalog.setInitialized(true);
+ }
// create db
ops = new IcebergMetadataOps(icebergCatalog,
icebergCatalog.getCatalog());
CreateDbStmt createDbStmt = new CreateDbStmt(true, new
DbName("iceberg", dbName), null);
ops.createDb(createDbStmt);
- icebergCatalog.setInitialized(true);
+ if (icebergCatalog.getUseMetaCache().get()) {
+ icebergCatalog.makeSureInitialized();
+ } else {
+ icebergCatalog.setInitialized(true);
+ }
IcebergExternalDatabase db = new
IcebergExternalDatabase(icebergCatalog, 1L, dbName);
icebergCatalog.addDatabaseForTest(db);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index 9050035f7c8..90a15db4aaa 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -265,10 +265,10 @@ public class PropertyConverterTest extends
TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(queryOld);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt,
"hms_s3_old");
Map<String, String> properties =
catalog.getCatalogProperty().getProperties();
- Assertions.assertEquals(properties.size(), 12);
+ Assertions.assertEquals(13, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), 21);
+ Assertions.assertEquals(22, hdProps.size());
}
@Test
@@ -283,10 +283,10 @@ public class PropertyConverterTest extends
TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(query);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt,
"hms_s3");
Map<String, String> properties =
catalog.getCatalogProperty().getProperties();
- Assertions.assertEquals(properties.size(), 11);
+ Assertions.assertEquals(12, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), 20);
+ Assertions.assertEquals(21, hdProps.size());
}
@Test
@@ -434,11 +434,11 @@ public class PropertyConverterTest extends
TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(queryOld);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt,
catalogName);
Map<String, String> properties = catalog.getProperties();
- Assertions.assertEquals(properties.size(), 20);
+ Assertions.assertEquals(properties.size(), 21);
Assertions.assertEquals("s3.us-east-1.amazonaws.com",
properties.get(S3Properties.ENDPOINT));
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), 29);
+ Assertions.assertEquals(30, hdProps.size());
String query = "create catalog hms_glue properties (\n"
+ " 'type'='hms',\n"
@@ -452,11 +452,11 @@ public class PropertyConverterTest extends
TestWithFeService {
CreateCatalogStmt analyzedStmtNew = createStmt(query);
HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew,
catalogName);
Map<String, String> propertiesNew = catalogNew.getProperties();
- Assertions.assertEquals(propertiesNew.size(), 20);
+ Assertions.assertEquals(21, propertiesNew.size());
Assertions.assertEquals("s3.us-east-1.amazonaws.com.cn",
propertiesNew.get(S3Properties.ENDPOINT));
Map<String, String> hdPropsNew =
catalogNew.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdPropsNew.size(), 29);
+ Assertions.assertEquals(30, hdPropsNew.size());
}
@Test
@@ -470,7 +470,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'cos.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName0,
CosProperties.COS_PREFIX,
- "cos.ap-beijing.myqcloud.com", query0, 11, 16);
+ "cos.ap-beijing.myqcloud.com", query0, 12, 17);
String catalogName1 = "hms_oss";
String query1 = "create catalog " + catalogName1 + " properties (\n"
@@ -481,7 +481,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'oss.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName1,
OssProperties.OSS_PREFIX,
- "oss.oss-cn-beijing.aliyuncs.com", query1, 11, 16);
+ "oss.oss-cn-beijing.aliyuncs.com", query1, 12, 17);
String catalogName2 = "hms_minio";
String query2 = "create catalog " + catalogName2 + " properties (\n"
@@ -492,7 +492,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'minio.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName2,
MinioProperties.MINIO_PREFIX,
- "http://127.0.0.1";, query2, 11, 20);
+ "http://127.0.0.1";, query2, 12, 21);
String catalogName3 = "hms_obs";
String query3 = "create catalog hms_obs properties (\n"
@@ -503,7 +503,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'obs.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName3,
ObsProperties.OBS_PREFIX,
- "obs.cn-north-4.myhuaweicloud.com", query3, 11, 16);
+ "obs.cn-north-4.myhuaweicloud.com", query3, 12, 17);
}
private void testS3CompatibleCatalogProperties(String catalogName, String
prefix,
@@ -513,10 +513,10 @@ public class PropertyConverterTest extends
TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(sql);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt,
catalogName);
Map<String, String> properties =
catalog.getCatalogProperty().getProperties();
- Assertions.assertEquals(properties.size(), catalogPropsSize);
+ Assertions.assertEquals(catalogPropsSize, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), bePropsSize);
+ Assertions.assertEquals(bePropsSize, hdProps.size());
Map<String, String> expectedMetaProperties = new HashMap<>();
expectedMetaProperties.put("endpoint", endpoint);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
index e90e1f8889f..8dd8421ebc2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Optional;
public class HmsCatalogTest extends AnalyzeCheckTestBase {
private static final String HMS_CATALOG = "hms_ctl";
@@ -90,6 +91,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) {
Deencapsulation.setField(hmsCatalog, "initialized", true);
Deencapsulation.setField(hmsCatalog, "objectCreated", true);
+ Deencapsulation.setField(hmsCatalog, "useMetaCache",
Optional.of(false));
List<Column> schema = Lists.newArrayList();
schema.add(new Column("k1", PrimitiveType.INT));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
index 949b28491d6..f342ca697b4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
@@ -51,6 +51,7 @@ import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.List;
+import java.util.Optional;
public class HmsQueryCacheTest extends AnalyzeCheckTestBase {
private static final String HMS_CATALOG = "hms_ctl";
@@ -109,6 +110,7 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase
{
private void init(HMSExternalCatalog hmsCatalog) {
Deencapsulation.setField(hmsCatalog, "initialized", true);
Deencapsulation.setField(hmsCatalog, "objectCreated", true);
+ Deencapsulation.setField(hmsCatalog, "useMetaCache",
Optional.of(false));
List<Column> schema = Lists.newArrayList();
schema.add(new Column("k1", PrimitiveType.INT));
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
new file mode 100644
index 00000000000..a5fc7ede82b
--- /dev/null
+++ b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
@@ -0,0 +1,77 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql01 --
+
+-- !sql02 --
+test_use_meta_cache_db
+
+-- !sql03 --
+test_use_meta_cache_tbl
+
+-- !sql04 --
+
+-- !sql05 --
+
+-- !sql01 --
+
+-- !sql02 --
+
+-- !sql03 --
+test_use_meta_cache_db_hive
+
+-- !sql04 --
+
+-- !sql05 --
+
+-- !sql06 --
+
+-- !sql07 --
+test_use_meta_cache_tbl_hive
+
+-- !sql08 --
+test_use_meta_cache_tbl_hive
+
+-- !sql09 --
+
+-- !sql10 --
+test_use_meta_cache_db_hive
+
+-- !sql11 --
+
+-- !sql01 --
+
+-- !sql02 --
+test_use_meta_cache_db
+
+-- !sql03 --
+test_use_meta_cache_tbl
+
+-- !sql04 --
+
+-- !sql05 --
+
+-- !sql01 --
+
+-- !sql02 --
+
+-- !sql03 --
+test_use_meta_cache_db_hive
+
+-- !sql04 --
+
+-- !sql05 --
+
+-- !sql06 --
+
+-- !sql07 --
+test_use_meta_cache_tbl_hive
+
+-- !sql08 --
+test_use_meta_cache_tbl_hive
+
+-- !sql09 --
+
+-- !sql10 --
+test_use_meta_cache_db_hive
+
+-- !sql11 --
+
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 8f123bb8d67..28b865ea6e9 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -814,6 +814,7 @@ class Suite implements GroovyInterceptable {
}
List<List<Object>> hive_docker(String sqlStr, boolean isOrder = false) {
+ logger.info("Execute hive ql: ${sqlStr}".toString())
String cleanedSqlStr = sqlStr.replaceAll("\\s*;\\s*\$", "")
def (result, meta) =
JdbcUtils.executeToList(context.getHiveDockerConnection(hivePrefix),
cleanedSqlStr)
if (isOrder) {
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
new file mode 100644
index 00000000000..3562ce31267
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_use_meta_cache",
"p0,external,hive,external_docker,external_docker_hive") {
+
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Hive test.")
+ return;
+ }
+
+ for (String hivePrefix : ["hive3", "hive3"]) {
+ setHivePrefix(hivePrefix)
+ try {
+ String hms_port = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String hdfs_port = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+ String catalog = "test_${hivePrefix}_use_meta_cache"
+ String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog}"""
+ sql """create catalog if not exists ${catalog} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hms_port}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'use_meta_cache' = 'true'
+ );"""
+
+ // create from Doris, the cache will be filled immediately
+ String database= "test_use_meta_cache_db"
+ String table = "test_use_meta_cache_tbl"
+ String database_hive = "test_use_meta_cache_db_hive"
+ String table_hive = "test_use_meta_cache_tbl_hive"
+ sql "switch ${catalog}"
+ sql "drop database if exists ${database}"
+ sql "drop database if exists ${database_hive}"
+ order_qt_sql01 "show databases like '%${database}%'";
+ sql "drop database if exists ${database}"
+ sql "create database ${database}"
+ order_qt_sql02 "show databases like '%${database}%'";
+ sql "use ${database}"
+ sql "create table ${table} (k1 int)"
+ order_qt_sql03 "show tables"
+ sql "drop table ${table}"
+ order_qt_sql04 "show tables"
+ sql "drop database ${database}"
+ order_qt_sql05 "show databases like '%${database}%'";
+
+ // create from Hive, the cache has different behavior
+ order_qt_sql01 "show databases like '%${database_hive}%'";
+ hive_docker "drop database if exists ${database_hive}"
+ hive_docker "create database ${database_hive}"
+ // not see
+ order_qt_sql02 "show databases like '%${database_hive}%'";
+ // but can use
+ sql "use ${database_hive}"
+ sql "refresh catalog ${catalog}"
+ // can see
+ order_qt_sql03 "show databases like '%${database_hive}%'";
+ // show tables first to fill cache
+ order_qt_sql04 "show tables"
+ hive_docker "create table ${database_hive}.${table_hive} (k1 int)"
+ // not see
+ order_qt_sql05 "show tables"
+ // but can select
+ sql "select * from ${table_hive}"
+ // still not see
+ order_qt_sql06 "show tables"
+ sql "refresh database ${database_hive}"
+ // can see
+ order_qt_sql07 "show tables"
+ hive_docker "drop table ${database_hive}.${table_hive}"
+ // still can see
+ order_qt_sql08 "show tables"
+ sql "refresh database ${database_hive}"
+ // can not see
+ order_qt_sql09 "show tables"
+ hive_docker "drop database ${database_hive}"
+ // still can see
+ order_qt_sql10 "show databases like '%${database_hive}%'";
+ sql "refresh catalog ${catalog}"
+ // can not see
+ order_qt_sql11 "show databases like '%${database_hive}%'";
+ } finally {
+ }
+ }
+}
+
+
diff --git
a/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy
b/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy
index 0435a68b09a..cd0533d00d9 100644
---
a/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy
+++
b/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy
@@ -191,7 +191,7 @@ suite("test_hive_write_partitions",
"p0,external,hive,external_docker,external_d
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
- logger.info("diable Hive test.")
+ logger.info("disable Hive test.")
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
