This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 57e8aa36eba [fix](information_schema) Prune TABLE_COMMENT in tables
status scan for external tables (#65989)
57e8aa36eba is described below
commit 57e8aa36ebac600037852c1c564bc0a5648cad60
Author: Raghvendra Singh <[email protected]>
AuthorDate: Mon Oct 19 15:13:57 2026 +0530
[fix](information_schema) Prune TABLE_COMMENT in tables status scan for
external tables (#65989)
Related PR: #64933, #64263
Problem Summary:
`information_schema.tables` on an external Iceberg catalog issues one
blocking `loadTable` against the remote catalog for every table in the
catalog, even when the query only projects cheap columns like
`TABLE_SCHEMA`/`TABLE_NAME`. A listing of a few-thousand-table REST
catalog takes minutes and floods the catalog service with per-table GETs
(measured: 3,882 tables → 2m52s, 4,365 tables → 5m38s).
This is an interaction of two changes: #64263 made
`IcebergExternalTable.getComment()` read the comment from table metadata
(a remote table load), and #64933's required-columns pruning in
`listTableStatus` guards every status column except the
`status.setComment(...)` fill, which remained unconditional — so the one
column whose fill is a remote round-trip is also the one column that is
always computed.
Fix: guard the comment fill with the same
`needTableStatusColumn(requiredColumns, "TABLE_COMMENT")` check used by
the other status columns. Callers that do not project `TABLE_COMMENT`
skip the remote loads entirely; explicit `TABLE_COMMENT` projections
keep the #64263 behavior. When `required_columns` is not set (older BE),
behavior is unchanged (fill everything), consistent with the rest of the
pruning.
### Release note
Fix information_schema.tables scans on external catalogs loading every
table's metadata from the remote catalog when TABLE_COMMENT is not
projected.
Signed-off-by: Raghvendra Singh <[email protected]>
---
.../apache/doris/service/FrontendServiceImpl.java | 9 +-
.../doris/service/FrontendServiceImplTest.java | 118 +++++++++++++++++++++
.../test_tables_status_column_pruning.groovy | 33 ++++++
3 files changed, 159 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 03d5a2c9f8d..ee1e5fa9f19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -747,7 +747,14 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
TTableStatus status = new TTableStatus();
status.setName(table.getName());
status.setType(table.getMysqlType());
- status.setComment(table.getComment());
+ // TABLE_COMMENT is a required Thrift field, so it
must always be
+ // set — but for some external tables (e.g.
Iceberg since #64263)
+ // getComment() loads the full table metadata from
the remote
+ // catalog. Only pay that lookup when the scan
actually projects
+ // TABLE_COMMENT, same as the other pruned status
columns.
+
status.setComment(needTableStatusColumn(requiredColumns, "TABLE_COMMENT")
+ ? table.getComment()
+ : "");
if (needTableStatusColumn(requiredColumns,
"ENGINE")) {
status.setEngine(table.getEngine());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
index 6c2cf4aa08c..543ab7ebae7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
@@ -313,6 +313,124 @@ public class FrontendServiceImplTest extends
TestWithFeService {
}
}
+ @Test
+ public void testListTableStatusPrunesCommentLookupWhenNotProjected()
throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.prune_comment_status_column(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("prune_comment_status_column");
+ OlapTable spyTable = Mockito.spy(table);
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("prune_comment_status_column");
+ params.setRequiredColumns(Collections.singleton("UPDATE_TIME"));
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assertions.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ // comment is a required Thrift field: it must be set (empty
placeholder),
+ // and the response must pass Thrift validation without the lookup.
+ Assertions.assertTrue(status.isSetComment());
+ Assertions.assertEquals("", status.getComment());
+ status.validate();
+ Mockito.verify(spyTable, Mockito.never()).getComment();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
+ @Test
+ public void testListTableStatusFillsCommentWhenProjected() throws
Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.fill_comment_status_column(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("fill_comment_status_column");
+ OlapTable spyTable = Mockito.spy(table);
+ Mockito.doReturn("projected comment").when(spyTable).getComment();
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("fill_comment_status_column");
+ params.setRequiredColumns(Collections.singleton("TABLE_COMMENT"));
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assertions.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ Assertions.assertTrue(status.isSetComment());
+ Assertions.assertEquals("projected comment", status.getComment());
+ status.validate();
+ Mockito.verify(spyTable, Mockito.atLeastOnce()).getComment();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
+ @Test
+ public void testListTableStatusFillsCommentWhenRequiredColumnsUnset()
throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.legacy_comment_status_column(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("legacy_comment_status_column");
+ OlapTable spyTable = Mockito.spy(table);
+ Mockito.doReturn("legacy comment").when(spyTable).getComment();
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("legacy_comment_status_column");
+ // required_columns unset: an older BE — everything is filled.
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assertions.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ Assertions.assertTrue(status.isSetComment());
+ Assertions.assertEquals("legacy comment", status.getComment());
+ status.validate();
+ Mockito.verify(spyTable, Mockito.atLeastOnce()).getComment();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
@Test
public void
testListTableStatusPrunesAllOptionalColumnsWhenRequiredColumnsIsEmpty() throws
Exception {
String createOlapTblStmt = "CREATE TABLE
test.prune_all_optional_status_columns(\n"
diff --git
a/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
b/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
index e6a3245d580..77c83e2fa19 100644
---
a/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
+++
b/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
@@ -38,6 +38,7 @@ suite("test_tables_status_column_pruning") {
contains "col=TABLE_SCHEMA"
contains "col=UPDATE_TIME"
notContains "col=TABLE_NAME"
+ notContains "col=TABLE_COMMENT"
assertNoTabletStatusColumns.delegate = delegate
assertNoTabletStatusColumns.call()
}
@@ -93,5 +94,37 @@ suite("test_tables_status_column_pruning") {
contains "col=DATA_LENGTH"
contains "col=AVG_ROW_LENGTH"
contains "col=INDEX_LENGTH"
+ contains "col=TABLE_COMMENT"
+ }
+
+ // TABLE_COMMENT pruning: for external tables the comment fill can be a
+ // remote metadata load, so a projection without it must not request it.
+ explain {
+ verbose true
+ sql """
+ SELECT TABLE_SCHEMA, TABLE_NAME
+ FROM information_schema.tables
+ WHERE TABLE_SCHEMA = 'information_schema'
+ """
+ contains "TABLE: information_schema.tables"
+ contains "col=TABLE_SCHEMA"
+ contains "col=TABLE_NAME"
+ notContains "col=TABLE_COMMENT"
+ assertNoTabletStatusColumns.delegate = delegate
+ assertNoTabletStatusColumns.call()
+ }
+
+ explain {
+ verbose true
+ sql """
+ SELECT TABLE_NAME, TABLE_COMMENT
+ FROM information_schema.tables
+ WHERE TABLE_SCHEMA = 'information_schema'
+ """
+ contains "TABLE: information_schema.tables"
+ contains "col=TABLE_NAME"
+ contains "col=TABLE_COMMENT"
+ assertNoTabletStatusColumns.delegate = delegate
+ assertNoTabletStatusColumns.call()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
