This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dcc27718d9a [improvement](catalog) avoid loading every table for SHOW
TABLES on external catalogs (#66080)
dcc27718d9a is described below
commit dcc27718d9a2db68536ccf3fee458708815256c2
Author: York Cao <[email protected]>
AuthorDate: Tue Aug 11 19:48:31 2026 +0800
[improvement](catalog) avoid loading every table for SHOW TABLES on
external catalogs (#66080)
### What problem does this PR solve?
Issue Number: close #66079
Problem Summary:
`SHOW TABLES` on an external catalog iterated `dbIf.getTables()`, which
eagerly initializes every table via the meta cache (one
`getTableNullable()`
-> remote metadata load per table). For catalogs with many tables this
turned
a cheap name listing into N remote loads.
This adds a fast path for the common case only -- a non-verbose `SHOW
TABLES`
on an external catalog -- that lists names via
`dbIf.getTableNamesOrEmptyWithLock()` without initializing any table.
The
per-table `SHOW` privilege filter is preserved (it is name-based and
needs no
table load), so no table name is leaked to a user who lacks `SHOW` on
it.
Everything else keeps the original `getTables()` loop unchanged:
- Internal-catalog `SHOW TABLES` / `SHOW FULL TABLES`.
- External-catalog `SHOW FULL TABLES` (verbose): it still needs
`getMysqlType()`
and the storage-format columns, so it takes the original path and its
4-column output (`Table_type`, `Storage_format`,
`Inverted_index_storage_format`) is unchanged.
- `SHOW VIEWS` (needs `getEngine()` to filter views) and `SHOW STREAMS`.
### Release note
None (performance optimization; `SHOW TABLES` output for accessible
tables is
unchanged, `SHOW FULL TABLES` / `SHOW VIEWS` are unaffected).
### Check List (For Author)
- Test:
- Manual test on a live cluster with an HMS catalog: `SHOW TABLES` and
`SHOW TABLES LIKE` list names via the fast path; `SHOW FULL TABLES`
returns the correct 4 columns (name, `Table_type`, `Storage_format`,
`Inverted_index_storage_format`) with no column mismatch.
- Ran existing `ShowTableCommandTest` (2 passed); FE build (`build.sh
--fe`)
+ checkstyle green.
- No new unit test: exercising the external-catalog branch needs an
ExternalCatalog with a working `getTableNamesOrEmptyWithLock()`, which
is
not available in the FE unit-test harness; a portable regression `.out`
needs the docker HMS test environment.
- Behavior changed: No material change (a non-loadable external table's
name is
now listed by `SHOW TABLES` where the old path silently omitted it on
load
failure, which is arguably more correct).
- Does this need documentation: No
---------
Co-authored-by: lbs <[email protected]>
---
.../trees/plans/commands/ShowTableCommand.java | 21 +++
.../trees/plans/commands/ShowTableCommandTest.java | 171 +++++++++++++++++++++
2 files changed, 192 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java
index 0345e1ed30c..73368e914d9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java
@@ -31,6 +31,7 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.PatternMatcherWrapper;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.AliasInfo;
@@ -174,6 +175,26 @@ public class ShowTableCommand extends ShowCommand {
Preconditions.checkArgument(table.get().getType().equals(TableIf.TableType.STREAM));
rows.add(Lists.newArrayList(table.get().getName()));
}
+ } else if (!(dbIf.getCatalog() instanceof InternalCatalog)
+ && !isVerbose && type.equals(PlanType.SHOW_TABLES)) {
+ // Non-verbose SHOW TABLES on an external catalog: list names
directly
+ // instead of dbIf.getTables(), which loads every table via the
meta
+ // cache (one remote metadata load per table). The per-table SHOW
priv
+ // filter below must be kept (name-based, needs no table load).
+ // NOTE: must use getTableNamesWithLock(), NOT
getTableNamesOrEmptyWithLock():
+ // the latter swallows the case-insensitive name-conflict /
meta_names_mapping
+ // exception and returns an empty set, silently hiding conflicting
table names.
+ for (String tableName : dbIf.getTableNamesWithLock()) {
+ if (matcher != null && !matcher.match(tableName)) {
+ continue;
+ }
+ if (!Env.getCurrentEnv().getAccessManager()
+ .checkTblPriv(ConnectContext.get(), catalog,
dbIf.getFullName(), tableName,
+ PrivPredicate.SHOW)) {
+ continue;
+ }
+ rows.add(Lists.newArrayList(tableName));
+ }
} else {
for (TableIf tbl : dbIf.getTables()) {
if (type.equals(PlanType.SHOW_VIEWS) && (tbl.getEngine() ==
null
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommandTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommandTest.java
index 6f273d3df4d..f1715d9cbfc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommandTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommandTest.java
@@ -18,18 +18,35 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.backup.CatalogMocker;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.TestWithFeService;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import java.io.IOException;
+import java.util.List;
public class ShowTableCommandTest extends TestWithFeService {
+ private static final String CATALOG_NAME = "hive_catalog";
+ private static final String DB_NAME = "hive_db";
+
private ConnectContext ctx;
private void runBefore() throws IOException {
@@ -63,4 +80,158 @@ public class ShowTableCommandTest extends TestWithFeService
{
"", false, PlanType.SHOW_TABLES);
Assertions.assertThrows(AnalysisException.class, () ->
command2.validate(ctx));
}
+
+ /**
+ * Bundle of mocks needed to drive {@link ShowTableCommand#doRun}: a
mocked {@link ConnectContext}
+ * whose {@code
getEnv().getCatalogMgr().getCatalogOrAnalysisException(...).getDbOrAnalysisException(...)}
+ * chain resolves to {@code dbIf}, wired to the given catalog (which
decides whether
+ * {@code dbIf.getCatalog() instanceof InternalCatalog} holds).
+ */
+ private static final class ShowTableCommandMocks {
+ private final ConnectContext ctx = Mockito.mock(ConnectContext.class);
+ private final StmtExecutor executor = Mockito.mock(StmtExecutor.class);
+ private final Env env = Mockito.mock(Env.class);
+ private final AccessControllerManager accessControllerManager =
Mockito.mock(AccessControllerManager.class);
+ @SuppressWarnings("unchecked")
+ private final DatabaseIf<TableIf> dbIf =
Mockito.mock(DatabaseIf.class);
+
+ @SuppressWarnings("unchecked")
+ ShowTableCommandMocks(CatalogIf<?> catalogIf) throws AnalysisException
{
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.when(ctx.getEnv()).thenReturn(env);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+
Mockito.when(env.getAccessManager()).thenReturn(accessControllerManager);
+
Mockito.when(catalogMgr.getCatalogOrAnalysisException(Mockito.anyString())).thenReturn(catalogIf);
+
Mockito.when(catalogIf.getDbOrAnalysisException(Mockito.anyString())).thenReturn(dbIf);
+ Mockito.when(dbIf.getCatalog()).thenReturn(catalogIf);
+ Mockito.when(dbIf.getFullName()).thenReturn(DB_NAME);
+ }
+ }
+
+ private static ShowResultSet runDoRun(ShowTableCommandMocks mocks,
ShowTableCommand command) throws Exception {
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
+ MockedStatic<ConnectContext> mockedConnectContext =
Mockito.mockStatic(ConnectContext.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(mocks.env);
+ // Lower-case-table-names off, so LIKE patterns are matched
case-sensitively.
+ mockedEnv.when(() ->
Env.getLowerCaseTableNames(Mockito.anyString())).thenReturn(0);
+
mockedConnectContext.when(ConnectContext::get).thenReturn(mocks.ctx);
+ return command.doRun(mocks.ctx, mocks.executor);
+ }
+ }
+
+ @Test
+ public void testExternalCatalogNonVerboseShowTablesUsesFastPath() throws
Exception {
+ CatalogIf<?> catalogIf = Mockito.mock(CatalogIf.class);
+ ShowTableCommandMocks mocks = new ShowTableCommandMocks(catalogIf);
+ Mockito.when(mocks.dbIf.getTableNamesWithLock())
+ .thenReturn(ImmutableSet.of("t2", "t1", "t_filtered_out"));
+ // Every table is visible except "t_filtered_out", which SHOW
privilege denies.
+ Mockito.when(mocks.accessControllerManager.checkTblPriv(
+ Mockito.eq(mocks.ctx), Mockito.eq(CATALOG_NAME),
Mockito.eq(DB_NAME),
+ Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW)))
+ .thenAnswer(invocation ->
!"t_filtered_out".equals(invocation.getArgument(3)));
+
+ ShowTableCommand command = new ShowTableCommand(DB_NAME, CATALOG_NAME,
false, PlanType.SHOW_TABLES);
+ ShowResultSet result = runDoRun(mocks, command);
+
+ // Behavior: names come back sorted, and the privilege-denied table is
excluded.
+ List<List<String>> rows = result.getResultRows();
+ Assertions.assertEquals(2, rows.size());
+ Assertions.assertEquals(Lists.newArrayList("t1"), rows.get(0));
+ Assertions.assertEquals(Lists.newArrayList("t2"), rows.get(1));
+
+ // Call counts: the fast path must list names, and must never load
every table.
+ // It uses getTableNamesWithLock() (not the exception-swallowing
...OrEmpty... variant)
+ // so a name-conflict exception raised while listing is still
propagated.
+ Mockito.verify(mocks.dbIf, Mockito.times(1)).getTableNamesWithLock();
+ Mockito.verify(mocks.dbIf, Mockito.times(0)).getTables();
+ }
+
+ @Test
+ public void testExternalCatalogShowTablesLikePatternUsesFastPath() throws
Exception {
+ CatalogIf<?> catalogIf = Mockito.mock(CatalogIf.class);
+ ShowTableCommandMocks mocks = new ShowTableCommandMocks(catalogIf);
+ Mockito.when(mocks.dbIf.getTableNamesWithLock())
+ .thenReturn(ImmutableSet.of("tbl1", "tbl2", "other_tbl"));
+ Mockito.when(mocks.accessControllerManager.checkTblPriv(
+ Mockito.eq(mocks.ctx), Mockito.eq(CATALOG_NAME),
Mockito.eq(DB_NAME),
+ Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW)))
+ .thenReturn(true);
+
+ // The mysql LIKE pattern is applied on the names-only fast path:
"tbl_"
+ // matches tbl1/tbl2, but not other_tbl.
+ ShowTableCommand command = new ShowTableCommand(DB_NAME, CATALOG_NAME,
false, "tbl_", null,
+ PlanType.SHOW_TABLES);
+ ShowResultSet result = runDoRun(mocks, command);
+
+ List<List<String>> rows = result.getResultRows();
+ Assertions.assertEquals(2, rows.size());
+ Assertions.assertEquals(Lists.newArrayList("tbl1"), rows.get(0));
+ Assertions.assertEquals(Lists.newArrayList("tbl2"), rows.get(1));
+ Mockito.verify(mocks.dbIf, Mockito.times(1)).getTableNamesWithLock();
+ Mockito.verify(mocks.dbIf, Mockito.times(0)).getTables();
+ }
+
+ @Test
+ public void testExternalCatalogShowTablesPropagatesNameListingFailure()
throws Exception {
+ CatalogIf<?> catalogIf = Mockito.mock(CatalogIf.class);
+ ShowTableCommandMocks mocks = new ShowTableCommandMocks(catalogIf);
+ // Guards the getTableNamesWithLock() contract: a case-insensitive
name-conflict
+ // failure must surface as an error, not be swallowed into an empty
result.
+ Mockito.when(mocks.dbIf.getTableNamesWithLock()).thenThrow(
+ new RuntimeException("Found conflicting table names under
case-insensitive conditions"));
+
+ ShowTableCommand command = new ShowTableCommand(DB_NAME, CATALOG_NAME,
false, PlanType.SHOW_TABLES);
+ Assertions.assertThrows(RuntimeException.class, () -> runDoRun(mocks,
command));
+ Mockito.verify(mocks.dbIf, Mockito.times(0)).getTables();
+ }
+
+ @Test
+ public void testExternalCatalogVerboseShowTablesUsesSlowPath() throws
Exception {
+ CatalogIf<?> catalogIf = Mockito.mock(CatalogIf.class);
+ ShowTableCommandMocks mocks = new ShowTableCommandMocks(catalogIf);
+ Mockito.when(mocks.dbIf.getTables()).thenReturn(Lists.newArrayList());
+
+ // Verbose SHOW TABLES needs per-table metadata (storage format,
etc.), so even on an
+ // external catalog it must fall back to the slow path that loads
every table.
+ ShowTableCommand command = new ShowTableCommand(DB_NAME, CATALOG_NAME,
true, PlanType.SHOW_TABLES);
+ ShowResultSet result = runDoRun(mocks, command);
+
+ Assertions.assertTrue(result.getResultRows().isEmpty());
+ Mockito.verify(mocks.dbIf, Mockito.times(1)).getTables();
+ Mockito.verify(mocks.dbIf, Mockito.times(0)).getTableNamesWithLock();
+ }
+
+ @Test
+ public void testExternalCatalogShowViewsUsesSlowPath() throws Exception {
+ CatalogIf<?> catalogIf = Mockito.mock(CatalogIf.class);
+ ShowTableCommandMocks mocks = new ShowTableCommandMocks(catalogIf);
+ Mockito.when(mocks.dbIf.getTables()).thenReturn(Lists.newArrayList());
+
+ // SHOW VIEWS needs the engine type of every table to filter views, so
the name-only
+ // fast path (guarded by PlanType.SHOW_TABLES) must not be taken here.
+ ShowTableCommand command = new ShowTableCommand(DB_NAME, CATALOG_NAME,
false, PlanType.SHOW_VIEWS);
+ ShowResultSet result = runDoRun(mocks, command);
+
+ Assertions.assertTrue(result.getResultRows().isEmpty());
+ Mockito.verify(mocks.dbIf, Mockito.times(1)).getTables();
+ Mockito.verify(mocks.dbIf, Mockito.times(0)).getTableNamesWithLock();
+ }
+
+ @Test
+ public void testInternalCatalogNonVerboseShowTablesUsesSlowPath() throws
Exception {
+ InternalCatalog internalCatalog = Mockito.mock(InternalCatalog.class);
+ ShowTableCommandMocks mocks = new
ShowTableCommandMocks(internalCatalog);
+ Mockito.when(mocks.dbIf.getTables()).thenReturn(Lists.newArrayList());
+
+ // The fast path is only for external catalogs; internal catalogs
always take the
+ // slow path regardless of verbosity.
+ ShowTableCommand command = new ShowTableCommand(DB_NAME,
InternalCatalog.INTERNAL_CATALOG_NAME, false,
+ PlanType.SHOW_TABLES);
+ ShowResultSet result = runDoRun(mocks, command);
+
+ Assertions.assertTrue(result.getResultRows().isEmpty());
+ Mockito.verify(mocks.dbIf, Mockito.times(1)).getTables();
+ Mockito.verify(mocks.dbIf, Mockito.times(0)).getTableNamesWithLock();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
