wenzhenghu commented on code in PR #64035:
URL: https://github.com/apache/doris/pull/64035#discussion_r3348814909
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java:
##########
@@ -992,6 +1045,172 @@ public void setSnapshot(MvccTableInfo mvccTableInfo,
MvccSnapshot snapshot) {
snapshots.put(mvccTableInfo, snapshot);
}
+ private Optional<String> getExternalMetadataPreloadSkipReason() {
+ if (connectContext == null || connectContext.getSessionVariable() ==
null
+ ||
!connectContext.getSessionVariable().isEnablePreloadExternalMetadata()) {
+ return Optional.of("session variable
enable_preload_external_metadata is disabled");
+ }
+ if (externalTablePreloadInfos.isEmpty()) {
+ return Optional.of("no external preload candidates were
collected");
+ }
+ if (containsPlanReadLockTable(tables.values())
+ || containsPlanReadLockTable(mtmvRelatedTables.values())
+ || containsPlanReadLockTable(insertTargetTables.values())) {
+ return Optional.empty();
+ }
+ return Optional.of("no internal tables require plan-time read lock");
+ }
+
+ private boolean containsPlanReadLockTable(Collection<TableIf> tableIfs) {
+ for (TableIf tableIf : tableIfs) {
+ if (tableIf.needReadLockWhenPlan()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean supportsExternalMetadataPreload(TableIf table) {
+ return table instanceof HMSExternalTable
+ || table instanceof IcebergExternalTable
+ || table instanceof PaimonExternalTable
+ // Limit plugin-driven preload to JDBC catalogs until other
connectors are verified.
+ || isJdbcPluginExternalTable(table);
+ }
+
+ // Preload metadata that is commonly accessed during planning before
internal table locks are acquired.
+ private boolean preloadExternalTable(ExternalTablePreloadInfo preloadInfo)
{
+ ExternalTable table = preloadInfo.table;
+ long preloadStartTime = TimeUtils.getStartTimeMs();
+ // Preload the latest snapshot only when every relation uses the
latest view of the table.
+ boolean supportsLatestSnapshot = supportsLatestSnapshotPreload(table);
+ boolean latestOnlyRelation = preloadInfo.shouldPreloadLatestSnapshot();
+ boolean preloadLatestSnapshot = latestOnlyRelation &&
supportsLatestSnapshot;
+ // Skip schema and partition warmup for snapshot-aware tables when the
query targets
+ // only non-latest relations such as time travel, branch, or tag
references.
+ boolean preloadSchema = !supportsLatestSnapshot || latestOnlyRelation;
+ boolean preloadPartition = preloadSchema &&
table.supportInternalPartitionPruned();
+ if (preloadLatestSnapshot) {
+ loadSnapshots(table, Optional.empty(), Optional.empty());
+ }
+ // Preload schema access while no internal table lock is held.
+ if (preloadSchema) {
+ table.getBaseSchema();
+ }
+ // Preload partition metadata only when the warmed schema matches the
relation view.
+ if (preloadPartition) {
+ table.initSelectedPartitions(getSnapshot(table));
+ }
+ // Log the actual preload path per table to simplify manual
verification in debug mode.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} preloaded external metadata for table {} "
+ + "[supportsLatestSnapshot={},
preloadLatestSnapshot={}, preloadSchema={}, "
+ + "preloadPartition={}, hasLatestRelation={},
hasNonLatestRelation={}, elapsedMs={}]",
+ getPreloadQueryIdentifier(),
getExternalTableLogName(table), supportsLatestSnapshot,
+ preloadLatestSnapshot, preloadSchema, preloadPartition,
preloadInfo.hasLatestOnlyRelation,
+ preloadInfo.hasNonLatestRelation,
TimeUtils.getElapsedTimeMs(preloadStartTime));
+ }
+ return preloadLatestSnapshot || preloadSchema || preloadPartition;
+ }
+
+ // Use the query identifier in debug logs so preload events can be
correlated with a single statement.
+ private String getPreloadQueryIdentifier() {
+ return connectContext == null ? "stmt[unknown]" :
connectContext.getQueryIdentifier();
+ }
+
+ // Build a stable table name for preload debug logs.
+ private String getExternalTableLogName(ExternalTable table) {
+ return table.getCatalog().getName() + "." + table.getDbName() + "." +
table.getName();
+ }
+
+ // Restrict plugin-driven preload to JDBC because other connector types
have not been evaluated yet.
+ private boolean isJdbcPluginExternalTable(TableIf table) {
+ if (!(table instanceof PluginDrivenExternalTable)) {
+ return false;
+ }
+ ExternalTable externalTable = (ExternalTable) table;
+ if (!(externalTable.getCatalog() instanceof
PluginDrivenExternalCatalog)) {
+ return false;
+ }
+ return "jdbc".equalsIgnoreCase(((PluginDrivenExternalCatalog)
externalTable.getCatalog()).getType());
+ }
+
+ // Keep the latest snapshot preload decision explicit so metadata preload
support and
+ // snapshot preload support are not conflated.
+ private boolean supportsLatestSnapshotPreload(ExternalTable table) {
+ // Paimon and Iceberg are not HMSExternalTable even when their catalog
metadata comes from HMS.
+ if (table instanceof PaimonExternalTable || table instanceof
IcebergExternalTable) {
+ return true;
+ }
+ if (!(table instanceof HMSExternalTable)) {
+ return false;
+ }
+ // Hive and Hudi share HMSExternalTable, but only Hudi has a
meaningful latest snapshot to preload.
+ DLAType dlaType = ((HMSExternalTable) table).getDlaType();
+ return dlaType == DLAType.HUDI;
+ }
+
+ private static class ExternalTablePreloadInfo {
Review Comment:
Good point. I will move ExternalTablePreloadInfo out of StatementContext
into a dedicated class to keep StatementContext lighter and avoid adding
another helper type in this file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
