This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 04a70ecaf01 [fix](info_db) do not fetch external catalog's info in
information_schema (#25844)
04a70ecaf01 is described below
commit 04a70ecaf010c488bb13591ef62cc92b0c964053
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Oct 25 21:34:36 2023 +0800
[fix](info_db) do not fetch external catalog's info in information_schema
(#25844)
There is FE config `infodb_support_ext_catalog`, the default is false.
Which means that the tables in `information_schema` database will not
return info of external catalog.
Because if there are too many external catalogs in Doris with lots of
db/tbl (like running p0 regression tests),
querying infomation_schema db will take a long time and may causing rpc
timeout.
And there is an unresolved issue that if thrift rpc timeout, the BE may be
crashed in ASAN mode.
So to avoid this issue(not fix yet), this PR mainly changes:
if `infodb_support_ext_catalog` is false,
1. query info of external catalog in information_schema db is not allowed,
such as
show database like "external_catalog";
show tables like "xxx"
2. select * from information_schema.tbl will not contains external
catalogs' info
3. For external p0 regression test pipeline, set
`infodb_support_ext_catalog` to true to run the tests related to external
catalog
---
.../apache/doris/service/FrontendServiceImpl.java | 41 +++++++++++++++++++---
regression-test/pipeline/external/conf/fe.conf | 1 +
2 files changed, 37 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index d9d0a03f3d0..febd638e73d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -378,12 +378,23 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
Env env = Env.getCurrentEnv();
List<CatalogIf> catalogIfs = Lists.newArrayList();
- if (Strings.isNullOrEmpty(params.catalog)) {
- catalogIfs = env.getCatalogMgr().listCatalogs();
+ // If infodb_support_ext_catalog is true, we will list all catalogs or
the specified catalog.
+ // Otherwise, we will only list internal catalog, or if the specified
catalog is internal catalog.
+ if (Config.infodb_support_ext_catalog) {
+ if (Strings.isNullOrEmpty(params.catalog)) {
+ catalogIfs = env.getCatalogMgr().listCatalogs();
+ } else {
+ catalogIfs.add(env.getCatalogMgr()
+ .getCatalogOrException(params.catalog,
+ catalog -> new TException("Unknown catalog " +
catalog)));
+ }
} else {
- catalogIfs.add(env.getCatalogMgr()
- .getCatalogOrException(params.catalog, catalog -> new
TException("Unknown catalog " + catalog)));
+ if (Strings.isNullOrEmpty(params.catalog)
+ ||
params.catalog.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ catalogIfs.add(env.getInternalCatalog());
+ }
}
+
for (CatalogIf catalog : catalogIfs) {
Collection<DatabaseIf> dbs = new HashSet<DatabaseIf>();
try {
@@ -621,6 +632,11 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
}
String catalogName = Strings.isNullOrEmpty(params.catalog) ?
InternalCatalog.INTERNAL_CATALOG_NAME
: params.catalog;
+ if (!Config.infodb_support_ext_catalog
+ &&
!catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ throw new TException("Not support getting external catalog info
when "
+ + "infodb_support_ext_catalog is false");
+ }
DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(catalogName, catalog -> new
TException("Unknown catalog " + catalog))
@@ -676,6 +692,12 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
if (params.isSetCatalog()) {
catalogName = params.catalog;
}
+ if (!Config.infodb_support_ext_catalog
+ &&
!catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ throw new TException("Not support getting external catalog info
when "
+ + "infodb_support_ext_catalog is false");
+ }
+
CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
if (catalog != null) {
DatabaseIf db = catalog.getDbNullable(params.db);
@@ -731,7 +753,6 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
}
public TListTableMetadataNameIdsResult
listTableMetadataNameIds(TGetTablesParams params) throws TException {
-
LOG.debug("get list simple table request: {}", params);
TListTableMetadataNameIdsResult result = new
TListTableMetadataNameIdsResult();
@@ -890,6 +911,11 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
String catalogName = Strings.isNullOrEmpty(params.catalog) ?
InternalCatalog.INTERNAL_CATALOG_NAME
: params.catalog;
+ if (!Config.infodb_support_ext_catalog
+ &&
!catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ throw new TException("Not support getting external catalog info
when "
+ + "infodb_support_ext_catalog is false");
+ }
DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(catalogName, catalog -> new
TException("Unknown catalog " + catalog))
.getDbNullable(params.db);
@@ -960,6 +986,11 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
String catalogName = Strings.isNullOrEmpty(params.catalog) ?
InternalCatalog.INTERNAL_CATALOG_NAME
: params.catalog;
+ if (!Config.infodb_support_ext_catalog
+ &&
!catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ throw new TException("Not support getting external catalog info
when "
+ + "infodb_support_ext_catalog is false");
+ }
DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(catalogName, catalog -> new
TException("Unknown catalog " + catalog))
.getDbNullable(params.db);
diff --git a/regression-test/pipeline/external/conf/fe.conf
b/regression-test/pipeline/external/conf/fe.conf
index adc042357ca..1766cf1ddb6 100644
--- a/regression-test/pipeline/external/conf/fe.conf
+++ b/regression-test/pipeline/external/conf/fe.conf
@@ -87,3 +87,4 @@ dynamic_partition_check_interval_seconds=3
enable_feature_binlog=true
auth_token = 5ff161c3-2c08-4079-b108-26c8850b6598
+infodb_support_ext_catalog=true
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
