This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 49482338d7f [fix](catalog) fix filtered database when
use_meta_cache=true (#41939)
49482338d7f is described below
commit 49482338d7fc719f36454f685ce409f852e35930
Author: Rayner Chen <[email protected]>
AuthorDate: Thu Oct 17 14:24:47 2024 +0800
[fix](catalog) fix filtered database when use_meta_cache=true (#41939)
Followup #38244
When setting `use_meta_cache=true`, the `include_database_list` and
`exclude_database_list`
will have no effect. This PR fix it.
Also fix a bug, related to #40479.
if `include_database_list` and `exclude_database_list` is set, and `use`
a database which is excluded,
Doris should return "database not found" error
---
.../apache/doris/datasource/ExternalCatalog.java | 40 +++--
.../doris/datasource/ExternalCatalogTest.java | 196 +++++++++++++++++++++
2 files changed, 218 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 25329a1a829..34525410834 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -53,6 +53,7 @@ import org.apache.doris.datasource.metacache.MetaCache;
import org.apache.doris.datasource.operations.ExternalMetadataOps;
import org.apache.doris.datasource.paimon.PaimonExternalDatabase;
import org.apache.doris.datasource.property.PropertyConverter;
+import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.datasource.test.TestExternalDatabase;
import
org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
@@ -349,19 +350,8 @@ public abstract class ExternalCatalog
InitCatalogLog initCatalogLog = new InitCatalogLog();
initCatalogLog.setCatalogId(id);
initCatalogLog.setType(logType);
- List<String> allDatabases = getFilteredDatabaseNames();
- Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap();
- Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
- for (String dbName : allDatabases) {
- if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) &&
!dbName.equals(MysqlDb.DATABASE_NAME)) {
- // Exclude database map take effect with higher priority over
include database map
- if (!excludeDatabaseMap.isEmpty() &&
excludeDatabaseMap.containsKey(dbName)) {
- continue;
- }
- if (!includeDatabaseMap.isEmpty() &&
!includeDatabaseMap.containsKey(dbName)) {
- continue;
- }
- }
+ List<String> filteredDatabases = getFilteredDatabaseNames();
+ for (String dbName : filteredDatabases) {
long dbId;
if (dbNameToId != null && dbNameToId.containsKey(dbName)) {
dbId = dbNameToId.get(dbName);
@@ -392,6 +382,20 @@ public abstract class ExternalCatalog
allDatabases.add(InfoSchemaDb.DATABASE_NAME);
allDatabases.remove(MysqlDb.DATABASE_NAME);
allDatabases.add(MysqlDb.DATABASE_NAME);
+ Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap();
+ Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
+ allDatabases = allDatabases.stream().filter(dbName -> {
+ if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) &&
!dbName.equals(MysqlDb.DATABASE_NAME)) {
+ // Exclude database map take effect with higher priority over
include database map
+ if (!excludeDatabaseMap.isEmpty() &&
excludeDatabaseMap.containsKey(dbName)) {
+ return false;
+ }
+ if (!includeDatabaseMap.isEmpty() &&
!includeDatabaseMap.containsKey(dbName)) {
+ return false;
+ }
+ }
+ return true;
+ }).collect(Collectors.toList());
return allDatabases;
}
@@ -675,11 +679,11 @@ public abstract class ExternalCatalog
InitCatalogLog.Type logType, boolean checkExists) {
// When running ut, disable this check to make ut pass.
// Because in ut, the database is not created in remote system.
- if (checkExists && !FeConstants.runningUnitTest) {
+ if (checkExists && (!FeConstants.runningUnitTest || this instanceof
TestExternalCatalog)) {
try {
List<String> dbNames = getDbNames();
if (!dbNames.contains(dbName)) {
- dbNames = listDatabaseNames();
+ dbNames = getFilteredDatabaseNames();
if (!dbNames.contains(dbName)) {
return null;
}
@@ -835,15 +839,15 @@ public abstract class ExternalCatalog
throw new NotImplementedException("registerDatabase not implemented");
}
- public Map<String, Boolean> getIncludeDatabaseMap() {
+ protected Map<String, Boolean> getIncludeDatabaseMap() {
return getSpecifiedDatabaseMap(Resource.INCLUDE_DATABASE_LIST);
}
- public Map<String, Boolean> getExcludeDatabaseMap() {
+ protected Map<String, Boolean> getExcludeDatabaseMap() {
return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST);
}
- public Map<String, Boolean> getSpecifiedDatabaseMap(String
catalogPropertyKey) {
+ private Map<String, Boolean> getSpecifiedDatabaseMap(String
catalogPropertyKey) {
String specifiedDatabaseList =
catalogProperty.getOrDefault(catalogPropertyKey, "");
Map<String, Boolean> specifiedDatabaseMap = Maps.newHashMap();
specifiedDatabaseList = specifiedDatabaseList.trim();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
index f527da2725e..43348ca8a0e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
@@ -17,16 +17,108 @@
package org.apache.doris.datasource;
+import org.apache.doris.analysis.CreateCatalogStmt;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.test.TestExternalCatalog;
+import org.apache.doris.mysql.privilege.Auth;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.QueryState.MysqlStateType;
+import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.TestWithFeService;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
public class ExternalCatalogTest extends TestWithFeService {
+ private static Auth auth;
+ private static Env env;
+ private CatalogMgr mgr;
+ private ConnectContext rootCtx;
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ FeConstants.runningUnitTest = true;
+ mgr = Env.getCurrentEnv().getCatalogMgr();
+ rootCtx = createDefaultCtx();
+ env = Env.getCurrentEnv();
+ auth = env.getAuth();
+ // 1. create test catalog
+ CreateCatalogStmt testCatalog = (CreateCatalogStmt)
parseAndAnalyzeStmt(
+ "create catalog test1 properties(\n"
+ + " \"type\" = \"test\",\n"
+ + " \"catalog_provider.class\" "
+ + "=
\"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ + " \"include_database_list\" = \"db1\"\n"
+ + ");",
+ rootCtx);
+ env.getCatalogMgr().createCatalog(testCatalog);
+
+ testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
+ "create catalog test2 properties(\n"
+ + " \"type\" = \"test\",\n"
+ + " \"catalog_provider.class\" "
+ + "=
\"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ + " \"exclude_database_list\" = \"db1\"\n"
+ + ");",
+ rootCtx);
+ env.getCatalogMgr().createCatalog(testCatalog);
+
+ testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
+ "create catalog test3 properties(\n"
+ + " \"type\" = \"test\",\n"
+ + " \"catalog_provider.class\" "
+ + "=
\"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ + " \"include_database_list\" = \"db1\",\n"
+ + " \"exclude_database_list\" = \"db1\"\n"
+ + ");",
+ rootCtx);
+ env.getCatalogMgr().createCatalog(testCatalog);
+
+ // use_meta_cache=false
+ testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
+ "create catalog test4 properties(\n"
+ + " \"type\" = \"test\",\n"
+ + " \"use_meta_cache\" = \"false\",\n"
+ + " \"catalog_provider.class\" "
+ + "=
\"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ + " \"include_database_list\" = \"db1\"\n"
+ + ");",
+ rootCtx);
+ env.getCatalogMgr().createCatalog(testCatalog);
+
+ testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
+ "create catalog test5 properties(\n"
+ + " \"type\" = \"test\",\n"
+ + " \"use_meta_cache\" = \"false\",\n"
+ + " \"catalog_provider.class\" "
+ + "=
\"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ + " \"exclude_database_list\" = \"db1\"\n"
+ + ");",
+ rootCtx);
+ env.getCatalogMgr().createCatalog(testCatalog);
+
+ testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
+ "create catalog test6 properties(\n"
+ + " \"type\" = \"test\",\n"
+ + " \"use_meta_cache\" = \"false\",\n"
+ + " \"catalog_provider.class\" "
+ + "=
\"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ + " \"include_database_list\" = \"db1\",\n"
+ + " \"exclude_database_list\" = \"db1\"\n"
+ + ");",
+ rootCtx);
+ env.getCatalogMgr().createCatalog(testCatalog);
+ }
@Test
public void testExternalCatalogAutoAnalyze() throws Exception {
@@ -48,4 +140,108 @@ public class ExternalCatalogTest extends TestWithFeService
{
catalog.modifyCatalogProps(prop);
Assertions.assertTrue(catalog.enableAutoAnalyze());
}
+
+ @Test
+ public void testExternalCatalogFilteredDatabase() throws Exception {
+ // 1. use_meta_cache=true
+ TestExternalCatalog ctl = (TestExternalCatalog)
mgr.getCatalog("test1");
+ List<String> dbNames = ctl.getDbNames();
+ System.out.println(dbNames);
+ Assertions.assertEquals(3, dbNames.size());
+ Assertions.assertTrue(!dbNames.contains("db2"));
+
+ ctl = (TestExternalCatalog) mgr.getCatalog("test2");
+ // before get dbnames
+ String useDb = "use test2.db3";
+ StmtExecutor stmtExecutor = new StmtExecutor(rootCtx, useDb);
+ stmtExecutor.execute();
+
Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown
database 'db3'"));
+
+ dbNames = ctl.getDbNames();
+ System.out.println(dbNames);
+ Assertions.assertEquals(3, dbNames.size());
+ Assertions.assertTrue(!dbNames.contains("db1"));
+
+ ctl = (TestExternalCatalog) mgr.getCatalog("test3");
+ dbNames = ctl.getDbNames();
+ System.out.println(dbNames);
+ Assertions.assertEquals(2, dbNames.size());
+ Assertions.assertTrue(!dbNames.contains("db1"));
+ Assertions.assertTrue(!dbNames.contains("db2"));
+
+ // use non exist db
+ useDb = "use test2.db3";
+ stmtExecutor = new StmtExecutor(rootCtx, useDb);
+ stmtExecutor.execute();
+
Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown
database 'db3'"));
+
+ // use exist db
+ useDb = "use test2.db2";
+ stmtExecutor = new StmtExecutor(rootCtx, useDb);
+ stmtExecutor.execute();
+ Assertions.assertEquals(MysqlStateType.OK,
rootCtx.getState().getStateType());
+
+ // 2. use_meta_cache=false
+ ctl = (TestExternalCatalog) mgr.getCatalog("test4");
+ dbNames = ctl.getDbNames();
+ System.out.println(dbNames);
+ Assertions.assertEquals(3, dbNames.size());
+ Assertions.assertTrue(!dbNames.contains("db2"));
+
+ ctl = (TestExternalCatalog) mgr.getCatalog("test5");
+ dbNames = ctl.getDbNames();
+ System.out.println(dbNames);
+ Assertions.assertEquals(3, dbNames.size());
+ Assertions.assertTrue(!dbNames.contains("db1"));
+
+ ctl = (TestExternalCatalog) mgr.getCatalog("test6");
+ dbNames = ctl.getDbNames();
+ System.out.println(dbNames);
+ Assertions.assertEquals(2, dbNames.size());
+ Assertions.assertTrue(!dbNames.contains("db1"));
+ Assertions.assertTrue(!dbNames.contains("db2"));
+
+ // use non exist db
+ useDb = "use test5.db3";
+ stmtExecutor = new StmtExecutor(rootCtx, useDb);
+ stmtExecutor.execute();
+
Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown
database 'db3'"));
+
+ // use exist db
+ useDb = "use test5.db2";
+ stmtExecutor = new StmtExecutor(rootCtx, useDb);
+ stmtExecutor.execute();
+ Assertions.assertEquals(MysqlStateType.OK,
rootCtx.getState().getStateType());
+ }
+
+ public static class RefreshCatalogProvider implements
TestExternalCatalog.TestCatalogProvider {
+ public static final Map<String, Map<String, List<Column>>> MOCKED_META;
+
+ static {
+ MOCKED_META = Maps.newHashMap();
+ Map<String, List<Column>> tblSchemaMap1 = Maps.newHashMap();
+ // db1
+ tblSchemaMap1.put("tbl11", Lists.newArrayList(
+ new Column("a11", PrimitiveType.BIGINT),
+ new Column("a12", PrimitiveType.STRING),
+ new Column("a13", PrimitiveType.FLOAT)));
+ tblSchemaMap1.put("tbl12", Lists.newArrayList(
+ new Column("b21", PrimitiveType.BIGINT),
+ new Column("b22", PrimitiveType.STRING),
+ new Column("b23", PrimitiveType.FLOAT)));
+ MOCKED_META.put("db1", tblSchemaMap1);
+ // db2
+ Map<String, List<Column>> tblSchemaMap2 = Maps.newHashMap();
+ tblSchemaMap2.put("tbl21", Lists.newArrayList(
+ new Column("c11", PrimitiveType.BIGINT),
+ new Column("c12", PrimitiveType.STRING),
+ new Column("c13", PrimitiveType.FLOAT)));
+ MOCKED_META.put("db2", tblSchemaMap2);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Column>>> getMetadata() {
+ return MOCKED_META;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
