This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1c1c66fecfe [improvement](fe) Add external table metadata profile
details (#63648)
1c1c66fecfe is described below
commit 1c1c66fecfee3842ea70db9f7264c9cff81f03bd
Author: Gabriel <[email protected]>
AuthorDate: Tue Jun 2 18:52:16 2026 +0800
[improvement](fe) Add external table metadata profile details (#63648)
Problem Summary: External table queries previously showed only coarse FE
metadata time in profile, making it hard to locate slow metadata access
steps for Hive, Iceberg, Hudi, and Paimon scans. This change records
dedicated profile timings for external partition value loading,
partition metadata loading, partition file listing, and file scan task
planning. The scan nodes keep a SummaryProfile reference so asynchronous
split planning can also report its metadata time.
---
.../doris/common/profile/SummaryProfile.java | 92 +++++++++++-
.../apache/doris/datasource/FileQueryScanNode.java | 14 +-
.../doris/datasource/hive/HMSExternalTable.java | 28 +++-
.../doris/datasource/hive/source/HiveScanNode.java | 83 ++++++----
.../apache/doris/datasource/hudi/HudiUtils.java | 77 ++++++----
.../doris/datasource/hudi/source/HudiScanNode.java | 167 ++++++++++++---------
.../iceberg/source/IcebergApiSource.java | 25 +--
.../iceberg/source/IcebergHMSSource.java | 10 +-
.../datasource/iceberg/source/IcebergScanNode.java | 82 ++++++----
.../datasource/paimon/source/PaimonScanNode.java | 57 ++++---
.../doris/common/profile/SummaryProfileTest.java | 24 +++
11 files changed, 455 insertions(+), 204 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
index 3ad317792a5..2b71352ba83 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
@@ -82,8 +82,12 @@ public class SummaryProfile {
public static final String INIT_SCAN_NODE_TIME = "Init Scan Node Time";
public static final String FINALIZE_SCAN_NODE_TIME = "Finalize Scan Node
Time";
public static final String GET_SPLITS_TIME = "Get Splits Time";
+ public static final String EXTERNAL_TABLE_META_TIME = "External Table Meta
Time";
+ public static final String EXTERNAL_TABLE_GET_TABLE_META_TIME = "External
Table Get Table Meta Time";
+ public static final String EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME =
"External Table Get Partition Values Time";
public static final String GET_PARTITIONS_TIME = "Get Partitions Time";
public static final String GET_PARTITION_FILES_TIME = "Get Partition Files
Time";
+ public static final String EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME =
"External Table Get File Scan Tasks Time";
public static final String CREATE_SCAN_RANGE_TIME = "Create Scan Range
Time";
public static final String SINK_SET_PARTITION_VALUES_TIME = "Sink Set
Partition Values Time";
public static final String PLAN_TIME = "Plan Time";
@@ -192,8 +196,12 @@ public class SummaryProfile {
INIT_SCAN_NODE_TIME,
FINALIZE_SCAN_NODE_TIME,
GET_SPLITS_TIME,
+ EXTERNAL_TABLE_META_TIME,
+ EXTERNAL_TABLE_GET_TABLE_META_TIME,
+ EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME,
GET_PARTITIONS_TIME,
GET_PARTITION_FILES_TIME,
+ EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME,
SINK_SET_PARTITION_VALUES_TIME,
CREATE_SCAN_RANGE_TIME,
ICEBERG_SCAN_METRICS,
@@ -243,10 +251,14 @@ public class SummaryProfile {
.put(INIT_SCAN_NODE_TIME, 2)
.put(FINALIZE_SCAN_NODE_TIME, 2)
.put(GET_SPLITS_TIME, 3)
+ .put(EXTERNAL_TABLE_META_TIME, 4)
+ .put(EXTERNAL_TABLE_GET_TABLE_META_TIME, 5)
+ .put(EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, 5)
.put(NEREIDS_DISTRIBUTE_TIME, 1)
.put(NEREIDS_BE_FOLD_CONST_TIME, 2)
.put(GET_PARTITIONS_TIME, 3)
.put(GET_PARTITION_FILES_TIME, 3)
+ .put(EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, 5)
.put(SINK_SET_PARTITION_VALUES_TIME, 3)
.put(CREATE_SCAN_RANGE_TIME, 2)
.put(ICEBERG_SCAN_METRICS, 3)
@@ -403,6 +415,26 @@ public class SummaryProfile {
private long nereidsMvRewriteTime = 0;
@SerializedName(value = "externalCatalogMetaTime")
private long externalCatalogMetaTime = 0;
+ // Total time to get table meta, including time to get table meta from
external catalog and time to do some
+ // process based on the meta, such as partition prune.
+ @SerializedName(value = "externalTableGetTableMetaTime")
+ private long externalTableGetTableMetaTime = 0;
+ // Total time to get partition values, including time to get partition
values from external catalog and time to do
+ // some process based on the partition values, such as partition prune.
+ @SerializedName(value = "externalTableGetPartitionValuesTime")
+ private long externalTableGetPartitionValuesTime = 0;
+ // Total time to get partitions, including time to get partitions from
external catalog and time to do some
+ // process based on the partitions, such as partition prune.
+ @SerializedName(value = "externalTableGetPartitionsTime")
+ private long externalTableGetPartitionsTime = 0;
+ // Total time to get partition files, including time to get partition
files from external catalog and time to do
+ // some process based on the partition files, such as creating scan range.
+ @SerializedName(value = "externalTableGetPartitionFilesTime")
+ private long externalTableGetPartitionFilesTime = 0;
+ // Total time to get file scan tasks, including time to get file scan
tasks from external catalog and time to do
+ // some process based on the file scan tasks, such as creating scan range.
+ @SerializedName(value = "externalTableGetFileScanTasksTime")
+ private long externalTableGetFileScanTasksTime = 0;
@SerializedName(value = "externalTvfInitTime")
private long externalTvfInitTime = 0;
@SerializedName(value = "nereidsPartitiionPruneTime")
@@ -562,10 +594,19 @@ public class SummaryProfile {
getPrettyTime(finalizeScanNodeFinishTime,
finalizeScanNodeStartTime, TUnit.TIME_MS));
executionSummaryProfile.addInfoString(GET_SPLITS_TIME,
getPrettyTime(getSplitsFinishTime, getSplitsStartTime,
TUnit.TIME_MS));
+ executionSummaryProfile.addInfoString(EXTERNAL_TABLE_META_TIME,
+ getPrettyAccumulatedTime(externalCatalogMetaTime));
+
executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_TABLE_META_TIME,
+ getPrettyAccumulatedTime(externalTableGetTableMetaTime));
+
executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME,
+ getPrettyAccumulatedTime(externalTableGetPartitionValuesTime));
executionSummaryProfile.addInfoString(GET_PARTITIONS_TIME,
- getPrettyTime(getPartitionsFinishTime, getSplitsStartTime,
TUnit.TIME_MS));
+ getExternalTableMetaTime(externalTableGetPartitionsTime,
getPartitionsFinishTime, getSplitsStartTime));
executionSummaryProfile.addInfoString(GET_PARTITION_FILES_TIME,
- getPrettyTime(getPartitionFilesFinishTime,
getPartitionsFinishTime, TUnit.TIME_MS));
+ getExternalTableMetaTime(externalTableGetPartitionFilesTime,
+ getPartitionFilesFinishTime, getPartitionsFinishTime));
+
executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME,
+ getPrettyAccumulatedTime(externalTableGetFileScanTasksTime));
executionSummaryProfile.addInfoString(SINK_SET_PARTITION_VALUES_TIME,
getPrettyTime(sinkSetPartitionValuesFinishTime,
sinkSetPartitionValuesStartTime, TUnit.TIME_MS));
executionSummaryProfile.addInfoString(CREATE_SCAN_RANGE_TIME,
@@ -1026,6 +1067,20 @@ public class SummaryProfile {
return RuntimeProfile.printCounter(end - start, unit);
}
+ private String getPrettyAccumulatedTime(long timeMs) {
+ if (timeMs <= 0) {
+ return "N/A";
+ }
+ return RuntimeProfile.printCounter(timeMs, TUnit.TIME_MS);
+ }
+
+ private String getExternalTableMetaTime(long accumulatedTimeMs, long end,
long start) {
+ if (accumulatedTimeMs > 0) {
+ return RuntimeProfile.printCounter(accumulatedTimeMs,
TUnit.TIME_MS);
+ }
+ return getPrettyTime(end, start, TUnit.TIME_MS);
+ }
+
public void setTransactionBeginTime(TransactionType type) {
this.transactionCommitBeginTime = TimeUtils.getStartTimeMs();
this.transactionType = type;
@@ -1191,6 +1246,39 @@ public class SummaryProfile {
return externalCatalogMetaTime;
}
+ public synchronized void addExternalTableGetTableMetaTime(long ms) {
+ this.externalTableGetTableMetaTime += ms;
+ addExternalCatalogMetaTimeInternal(ms);
+ }
+
+ public synchronized void addExternalTableGetPartitionValuesTime(long ms) {
+ this.externalTableGetPartitionValuesTime += ms;
+ addExternalCatalogMetaTimeInternal(ms);
+ }
+
+ public synchronized void addExternalTableGetPartitionsTime(long ms) {
+ this.externalTableGetPartitionsTime += ms;
+ addExternalCatalogMetaTimeInternal(ms);
+ }
+
+ public synchronized void addExternalTableGetPartitionFilesTime(long ms) {
+ this.externalTableGetPartitionFilesTime += ms;
+ addExternalCatalogMetaTimeInternal(ms);
+ }
+
+ public synchronized void addExternalTableGetFileScanTasksTime(long ms) {
+ this.externalTableGetFileScanTasksTime += ms;
+ addExternalCatalogMetaTimeInternal(ms);
+ }
+
+ public synchronized void addExternalCatalogMetaTime(long ms) {
+ addExternalCatalogMetaTimeInternal(ms);
+ }
+
+ private void addExternalCatalogMetaTimeInternal(long ms) {
+ this.externalCatalogMetaTime += ms;
+ }
+
public void addExternalTvfInitTime(long ms) {
this.externalTvfInitTime += ms;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index d2e25eb9534..a0002cc21c4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -34,6 +34,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.hive.source.HiveSplit;
import org.apache.doris.planner.PlanNodeId;
@@ -104,6 +105,7 @@ public abstract class FileQueryScanNode extends
FileScanNode {
protected TableScanParams scanParams;
protected FileSplitter fileSplitter;
+ protected SummaryProfile summaryProfile;
// The data cache function only works for queries on Hive, Iceberg,
Hudi(via HMS), and Paimon tables.
// See: https://doris.incubator.apache.org/docs/dev/lakehouse/data-cache
@@ -130,11 +132,12 @@ public abstract class FileQueryScanNode extends
FileScanNode {
public void init() throws UserException {
super.init();
if (ConnectContext.get().getExecutor() != null) {
-
ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeStartTime();
+ summaryProfile =
ConnectContext.get().getExecutor().getSummaryProfile();
+ summaryProfile.setInitScanNodeStartTime();
}
doInitialize();
if (ConnectContext.get().getExecutor() != null) {
-
ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeFinishTime();
+ summaryProfile.setInitScanNodeFinishTime();
}
}
@@ -155,6 +158,13 @@ public abstract class FileQueryScanNode extends
FileScanNode {
sessionVariable.maxInitialSplitNum);
}
+ protected SummaryProfile getSummaryProfile() {
+ if (summaryProfile == null) {
+ summaryProfile =
SummaryProfile.getSummaryProfile(ConnectContext.get());
+ }
+ return summaryProfile;
+ }
+
// Init schema (Tuple/Slot) related params.
protected void initSchemaParams() throws UserException {
destSlotDescByName = Maps.newHashMap();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 03dc966fd77..dad5b632fa2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -32,6 +32,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheKey;
import org.apache.doris.datasource.SchemaCacheValue;
@@ -249,7 +250,15 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
protected synchronized void makeSureInitialized() {
super.makeSureInitialized();
if (!objectCreated) {
- remoteTable = loadHiveTable();
+ long startTime = System.currentTimeMillis();
+ try {
+ remoteTable = loadHiveTable();
+ } finally {
+ SummaryProfile summaryProfile =
SummaryProfile.getSummaryProfile(null);
+ if (summaryProfile != null) {
+
summaryProfile.addExternalTableGetTableMetaTime(System.currentTimeMillis() -
startTime);
+ }
+ }
if (remoteTable == null) {
throw new IllegalArgumentException("Hms table not exists,
table: " + getNameWithFullQualifiers());
} else {
@@ -1281,11 +1290,18 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
public HiveExternalMetaCache.HivePartitionValues
getHivePartitionValues(Optional<MvccSnapshot> snapshot) {
+ long startTime = System.currentTimeMillis();
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.hive(getCatalog().getId());
try {
List<Type> partitionColumnTypes =
this.getPartitionColumnTypes(snapshot);
- return cache.getPartitionValues(this, partitionColumnTypes);
+ HiveExternalMetaCache.HivePartitionValues partitionValues =
cache.getPartitionValues(this,
+ partitionColumnTypes);
+ SummaryProfile summaryProfile =
SummaryProfile.getSummaryProfile(null);
+ if (summaryProfile != null) {
+
summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis()
- startTime);
+ }
+ return partitionValues;
} catch (Exception e) {
if
(e.getMessage().contains(HiveExternalMetaCache.ERR_CACHE_INCONSISTENCY)) {
LOG.warn("Hive metastore cache inconsistency detected for
table: {}.{}.{}. "
@@ -1294,7 +1310,13 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableByEngine(
getCatalog().getId(), getMetaCacheEngine(),
getDbName(), getName());
List<Type> partitionColumnTypes =
this.getPartitionColumnTypes(snapshot);
- return cache.getPartitionValues(this, partitionColumnTypes);
+ HiveExternalMetaCache.HivePartitionValues partitionValues =
cache.getPartitionValues(this,
+ partitionColumnTypes);
+ SummaryProfile summaryProfile =
SummaryProfile.getSummaryProfile(null);
+ if (summaryProfile != null) {
+
summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis()
- startTime);
+ }
+ return partitionValues;
} else {
throw e;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 0beb45dd620..5d1bb33f59a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -143,41 +143,51 @@ public class HiveScanNode extends FileQueryScanNode {
}
protected List<HivePartition> getPartitions() throws AnalysisException {
+ long startTime = System.currentTimeMillis();
List<HivePartition> resPartitions = Lists.newArrayList();
- HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .hive(hmsTable.getCatalog().getId());
- List<Type> partitionColumnTypes =
hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable));
- if (!partitionColumnTypes.isEmpty()) {
- // partitioned table
- Collection<PartitionItem> partitionItems;
- // partitions has benn pruned by Nereids, in
PruneFileScanPartition,
- // so just use the selected partitions.
- this.totalPartitionNum = selectedPartitions.totalPartitionNum;
- partitionItems = selectedPartitions.selectedPartitions.values();
- Preconditions.checkNotNull(partitionItems);
- this.selectedPartitionNum = partitionItems.size();
-
- // get partitions from cache
- List<List<String>> partitionValuesList =
Lists.newArrayListWithCapacity(partitionItems.size());
- for (PartitionItem item : partitionItems) {
- partitionValuesList.add(
- ((ListPartitionItem)
item).getItems().get(0).getPartitionValuesAsStringListForHive());
+ try {
+ HiveExternalMetaCache cache =
Env.getCurrentEnv().getExtMetaCacheMgr()
+ .hive(hmsTable.getCatalog().getId());
+ List<Type> partitionColumnTypes =
+
hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable));
+ if (!partitionColumnTypes.isEmpty()) {
+ // partitioned table
+ Collection<PartitionItem> partitionItems;
+ // partitions has benn pruned by Nereids, in
PruneFileScanPartition,
+ // so just use the selected partitions.
+ this.totalPartitionNum = selectedPartitions.totalPartitionNum;
+ partitionItems =
selectedPartitions.selectedPartitions.values();
+ Preconditions.checkNotNull(partitionItems);
+ this.selectedPartitionNum = partitionItems.size();
+
+ // get partitions from cache
+ List<List<String>> partitionValuesList =
Lists.newArrayListWithCapacity(partitionItems.size());
+ for (PartitionItem item : partitionItems) {
+ partitionValuesList.add(
+ ((ListPartitionItem)
item).getItems().get(0).getPartitionValuesAsStringListForHive());
+ }
+ resPartitions = cache.getAllPartitionsWithCache(hmsTable,
partitionValuesList);
+ } else {
+ // non partitioned table, create a dummy partition to save
location and inputformat,
+ // so that we can unify the interface.
+ HivePartition dummyPartition = new
HivePartition(hmsTable.getOrBuildNameMapping(), true,
+ hmsTable.getRemoteTable().getSd().getInputFormat(),
+ hmsTable.getRemoteTable().getSd().getLocation(), null,
Maps.newHashMap());
+ this.totalPartitionNum = 1;
+ this.selectedPartitionNum = 1;
+ resPartitions.add(dummyPartition);
}
- resPartitions = cache.getAllPartitionsWithCache(hmsTable,
partitionValuesList);
- } else {
- // non partitioned table, create a dummy partition to save
location and inputformat,
- // so that we can unify the interface.
- HivePartition dummyPartition = new
HivePartition(hmsTable.getOrBuildNameMapping(), true,
- hmsTable.getRemoteTable().getSd().getInputFormat(),
- hmsTable.getRemoteTable().getSd().getLocation(), null,
Maps.newHashMap());
- this.totalPartitionNum = 1;
- this.selectedPartitionNum = 1;
- resPartitions.add(dummyPartition);
- }
- if (ConnectContext.get().getExecutor() != null) {
-
ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionsFinishTime();
+ if (ConnectContext.get().getExecutor() != null) {
+
getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis()
- startTime);
+ getSummaryProfile().setGetPartitionsFinishTime();
+ }
+ return resPartitions;
+ } catch (RuntimeException e) {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis()
- startTime);
+ }
+ throw e;
}
- return resPartitions;
}
@Override
@@ -221,6 +231,7 @@ public class HiveScanNode extends FileQueryScanNode {
Executor scheduleExecutor =
Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
String bindBrokerName = hmsTable.getCatalog().bindBrokerName();
AtomicInteger numFinishedPartitions = new AtomicInteger(0);
+ long startTime = System.currentTimeMillis();
CompletableFuture.runAsync(() -> {
for (HivePartition partition : prunedPartitions) {
if (batchException.get() != null || splitAssignment.isStop()) {
@@ -248,6 +259,10 @@ public class HiveScanNode extends FileQueryScanNode {
splitAssignment.setException(batchException.get());
}
if (numFinishedPartitions.incrementAndGet() ==
prunedPartitions.size()) {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetPartitionFilesTime(
+ System.currentTimeMillis() -
startTime);
+ }
splitAssignment.finishSchedule();
}
}
@@ -287,6 +302,7 @@ public class HiveScanNode extends FileQueryScanNode {
List<Split> allFiles, String bindBrokerName, int numBackends,
boolean isBatchMode) throws IOException, UserException {
List<FileCacheValue> fileCaches;
+ long startTime = System.currentTimeMillis();
if (hiveTransaction != null) {
try {
fileCaches = getFileSplitByTransaction(cache, partitions,
bindBrokerName);
@@ -301,6 +317,9 @@ public class HiveScanNode extends FileQueryScanNode {
fileCaches = cache.getFilesByPartitions(partitions, withCache,
partitions.size() > 1,
directoryLister, hmsTable);
}
+ if (!isBatchMode && getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetPartitionFilesTime(System.currentTimeMillis()
- startTime);
+ }
long targetFileSplitSize = determineTargetFileSplitSize(fileCaches,
isBatchMode);
if (tableSample != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
index fbfe7a0a4da..6c622e371c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
@@ -26,6 +26,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.datasource.SchemaCacheValue;
import org.apache.doris.datasource.TablePartitionValues;
import org.apache.doris.datasource.hive.HMSExternalTable;
@@ -268,9 +269,14 @@ public class HudiUtils {
}
public static long getLastTimeStamp(HMSExternalTable hmsTable) {
+ long startTime = System.currentTimeMillis();
HoodieTableMetaClient hudiClient = hmsTable.getHudiClient();
HoodieTimeline timeline =
hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
+ SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null);
+ if (summaryProfile != null) {
+
summaryProfile.addExternalTableGetTableMetaTime(System.currentTimeMillis() -
startTime);
+ }
if (!snapshotInstant.isPresent()) {
return 0L;
}
@@ -279,40 +285,49 @@ public class HudiUtils {
public static TablePartitionValues
getPartitionValues(Optional<TableSnapshot> tableSnapshot,
HMSExternalTable hmsTable) {
- TablePartitionValues partitionValues = new TablePartitionValues();
-
- HoodieTableMetaClient hudiClient = hmsTable.getHudiClient();
- HudiExternalMetaCache hudiExternalMetaCache =
- Env.getCurrentEnv().getExtMetaCacheMgr()
- .hudi(hmsTable.getCatalog().getId());
- boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition();
-
- if (tableSnapshot.isPresent()) {
- if (tableSnapshot.get().getType() ==
TableSnapshot.VersionType.VERSION) {
- // Hudi does not support `FOR VERSION AS OF`, please use `FOR
TIME AS OF`";
- return partitionValues;
- }
- String queryInstant =
tableSnapshot.get().getValue().replaceAll("[-: ]", "");
- try {
- partitionValues =
hmsTable.getCatalog().getExecutionAuthenticator().execute(() ->
-
hudiExternalMetaCache.getSnapshotPartitionValues(hmsTable, queryInstant,
useHiveSyncPartition));
- } catch (Exception e) {
- throw new
RuntimeException(ExceptionUtils.getRootCauseMessage(e), e);
- }
- } else {
- HoodieTimeline timeline =
hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
- Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
- if (!snapshotInstant.isPresent()) {
- return partitionValues;
+ long startTime = System.currentTimeMillis();
+ try {
+ TablePartitionValues partitionValues = new TablePartitionValues();
+
+ HoodieTableMetaClient hudiClient = hmsTable.getHudiClient();
+ HudiExternalMetaCache hudiExternalMetaCache =
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .hudi(hmsTable.getCatalog().getId());
+ boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition();
+
+ if (tableSnapshot.isPresent()) {
+ if (tableSnapshot.get().getType() ==
TableSnapshot.VersionType.VERSION) {
+ // Hudi does not support `FOR VERSION AS OF`, please use
`FOR TIME AS OF`";
+ return partitionValues;
+ }
+ String queryInstant =
tableSnapshot.get().getValue().replaceAll("[-: ]", "");
+ try {
+ partitionValues =
hmsTable.getCatalog().getExecutionAuthenticator().execute(() ->
+ hudiExternalMetaCache.getSnapshotPartitionValues(
+ hmsTable, queryInstant,
useHiveSyncPartition));
+ } catch (Exception e) {
+ throw new
RuntimeException(ExceptionUtils.getRootCauseMessage(e), e);
+ }
+ } else {
+ HoodieTimeline timeline =
hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
+ Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
+ if (!snapshotInstant.isPresent()) {
+ return partitionValues;
+ }
+ try {
+ partitionValues =
hmsTable.getCatalog().getExecutionAuthenticator().execute(()
+ ->
hudiExternalMetaCache.getPartitionValues(hmsTable, useHiveSyncPartition));
+ } catch (Exception e) {
+ throw new
RuntimeException(ExceptionUtils.getRootCauseMessage(e), e);
+ }
}
- try {
- partitionValues =
hmsTable.getCatalog().getExecutionAuthenticator().execute(()
- -> hudiExternalMetaCache.getPartitionValues(hmsTable,
useHiveSyncPartition));
- } catch (Exception e) {
- throw new
RuntimeException(ExceptionUtils.getRootCauseMessage(e), e);
+ return partitionValues;
+ } finally {
+ SummaryProfile summaryProfile =
SummaryProfile.getSummaryProfile(null);
+ if (summaryProfile != null) {
+
summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis()
- startTime);
}
}
- return partitionValues;
}
public static HoodieTableMetaClient buildHudiTableMetaClient(String
hudiBasePath, Configuration conf) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
index a5970cea767..f3ab4332464 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -170,60 +170,67 @@ public class HudiScanNode extends HiveScanNode {
initBackendPolicy();
initSchemaParams();
- hudiClient = hmsTable.getHudiClient();
- hudiClient.reloadActiveTimeline();
- basePath = hmsTable.getRemoteTable().getSd().getLocation();
- inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat();
- serdeLib =
hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib();
-
- if (scanParams != null && !scanParams.incrementalRead()) {
- // Only support incremental read
- throw new UserException("Not support function '" +
scanParams.getParamType() + "' in hudi table");
- }
- if (incrementalRead) {
- if (isCowTable) {
- try {
- Map<String, String> serd =
hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters();
- if ("true".equals(serd.get("hoodie.query.as.ro.table"))
- &&
hmsTable.getRemoteTable().getTableName().endsWith("_ro")) {
- // Incremental read RO table as RT table, I don't know
why?
- isCowTable = false;
- LOG.warn("Execute incremental read on RO table: {}",
hmsTable.getFullQualifiers());
+ long tableMetaStartTime = System.currentTimeMillis();
+ try {
+ hudiClient = hmsTable.getHudiClient();
+ hudiClient.reloadActiveTimeline();
+ basePath = hmsTable.getRemoteTable().getSd().getLocation();
+ inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat();
+ serdeLib =
hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib();
+
+ if (scanParams != null && !scanParams.incrementalRead()) {
+ // Only support incremental read
+ throw new UserException("Not support function '" +
scanParams.getParamType() + "' in hudi table");
+ }
+ if (incrementalRead) {
+ if (isCowTable) {
+ try {
+ Map<String, String> serd =
hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters();
+ if ("true".equals(serd.get("hoodie.query.as.ro.table"))
+ &&
hmsTable.getRemoteTable().getTableName().endsWith("_ro")) {
+ // Incremental read RO table as RT table, I don't
know why?
+ isCowTable = false;
+ LOG.warn("Execute incremental read on RO table:
{}", hmsTable.getFullQualifiers());
+ }
+ } catch (Exception e) {
+ // ignore
}
- } catch (Exception e) {
- // ignore
+ }
+ if (incrementalRelation == null) {
+ throw new UserException("Failed to create incremental
relation");
}
}
- if (incrementalRelation == null) {
- throw new UserException("Failed to create incremental
relation");
- }
- }
- timeline =
hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
- TableSnapshot tableSnapshot = getQueryTableSnapshot();
- if (tableSnapshot != null) {
- if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) {
- throw new UserException("Hudi does not support `FOR VERSION AS
OF`, please use `FOR TIME AS OF`");
+ timeline =
hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
+ TableSnapshot tableSnapshot = getQueryTableSnapshot();
+ if (tableSnapshot != null) {
+ if (tableSnapshot.getType() ==
TableSnapshot.VersionType.VERSION) {
+ throw new UserException("Hudi does not support `FOR
VERSION AS OF`, please use `FOR TIME AS OF`");
+ }
+ queryInstant = tableSnapshot.getValue().replaceAll("[-: ]",
"");
+ } else {
+ Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
+ if (!snapshotInstant.isPresent()) {
+ prunedPartitions = Collections.emptyList();
+ partitionInit = true;
+ return;
+ }
+ queryInstant = snapshotInstant.get().requestedTime();
}
- queryInstant = tableSnapshot.getValue().replaceAll("[-: ]", "");
- } else {
- Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
- if (!snapshotInstant.isPresent()) {
- prunedPartitions = Collections.emptyList();
- partitionInit = true;
- return;
+
+ HudiSchemaCacheValue hudiSchemaCacheValue =
HudiUtils.getSchemaCacheValue(hmsTable, queryInstant);
+ columnNames =
hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList());
+ columnTypes = hudiSchemaCacheValue.getColTypes();
+
+ fsView = Env.getCurrentEnv()
+ .getExtMetaCacheMgr()
+ .hudi(hmsTable.getCatalog().getId())
+ .getFsView(hmsTable.getOrBuildNameMapping());
+ } finally {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis()
- tableMetaStartTime);
}
- queryInstant = snapshotInstant.get().requestedTime();
}
-
- HudiSchemaCacheValue hudiSchemaCacheValue =
HudiUtils.getSchemaCacheValue(hmsTable, queryInstant);
- columnNames =
hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList());
- columnTypes = hudiSchemaCacheValue.getColTypes();
-
- fsView = Env.getCurrentEnv()
- .getExtMetaCacheMgr()
- .hudi(hmsTable.getCatalog().getId())
- .getFsView(hmsTable.getOrBuildNameMapping());
// Todo: Get the current schema id of the table, instead of using -1.
// In Be Parquet/Rrc reader, if `current table schema id == current
file schema id`, then its
// `table_info_node_ptr` will be `TableSchemaChangeHelper::ConstNode`.
When using `ConstNode`,
@@ -368,21 +375,30 @@ public class HudiScanNode extends HiveScanNode {
}
private List<Split> getIncrementalSplits() {
+ long startTime = System.currentTimeMillis();
if (canUseNativeReader()) {
List<Split> splits = incrementalRelation.collectSplits();
noLogsSplitNum.addAndGet(splits.size());
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis()
- startTime);
+ }
return splits;
}
Option<String[]> partitionColumns =
hudiClient.getTableConfig().getPartitionFields();
List<String> partitionNames = partitionColumns.isPresent() ?
Arrays.asList(partitionColumns.get())
: Collections.emptyList();
- return incrementalRelation.collectFileSlices().stream().map(fileSlice
-> generateHudiSplit(fileSlice,
- HudiPartitionUtils.parsePartitionValues(partitionNames,
fileSlice.getPartitionPath()),
- incrementalRelation.getEndTs())).collect(Collectors.toList());
+ List<Split> splits = incrementalRelation.collectFileSlices().stream()
+ .map(fileSlice -> generateHudiSplit(fileSlice,
+
HudiPartitionUtils.parsePartitionValues(partitionNames,
fileSlice.getPartitionPath()),
+ incrementalRelation.getEndTs()))
+ .collect(Collectors.toList());
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis()
- startTime);
+ }
+ return splits;
}
private void getPartitionSplits(HivePartition partition, List<Split>
splits) throws IOException {
-
String partitionName;
if (partition.isDummyPartition()) {
partitionName = "";
@@ -422,6 +438,7 @@ public class HudiScanNode extends HiveScanNode {
Executor executor =
Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor();
CountDownLatch countDownLatch = new CountDownLatch(partitions.size());
AtomicReference<Throwable> throwable = new AtomicReference<>();
+ long startTime = System.currentTimeMillis();
partitions.forEach(partition -> executor.execute(() -> {
try {
getPartitionSplits(partition, splits);
@@ -439,6 +456,9 @@ public class HudiScanNode extends HiveScanNode {
if (throwable.get() != null) {
throw new RuntimeException(throwable.get().getMessage(),
throwable.get());
}
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis()
- startTime);
+ }
}
@Override
@@ -446,15 +466,7 @@ public class HudiScanNode extends HiveScanNode {
if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) {
return getIncrementalSplits();
}
- if (!partitionInit) {
- try {
- prunedPartitions =
hmsTable.getCatalog().getExecutionAuthenticator().execute(()
- -> getPrunedPartitions(hudiClient));
- } catch (Exception e) {
- throw new UserException(ExceptionUtils.getRootCauseMessage(e),
e);
- }
- partitionInit = true;
- }
+ initPrunedPartitions();
List<Split> splits = Collections.synchronizedList(new ArrayList<>());
try {
hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> {
@@ -467,6 +479,23 @@ public class HudiScanNode extends HiveScanNode {
return splits;
}
+ private void initPrunedPartitions() throws UserException {
+ if (partitionInit) {
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+ try {
+ prunedPartitions =
hmsTable.getCatalog().getExecutionAuthenticator().execute(()
+ -> getPrunedPartitions(hudiClient));
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis()
- startTime);
+ }
+ } catch (Exception e) {
+ throw new UserException(ExceptionUtils.getRootCauseMessage(e), e);
+ }
+ partitionInit = true;
+ }
+
@Override
public void startSplit(int numBackends) {
if (prunedPartitions.isEmpty()) {
@@ -475,6 +504,7 @@ public class HudiScanNode extends HiveScanNode {
}
AtomicInteger numFinishedPartitions = new AtomicInteger(0);
ExecutorService scheduleExecutor =
Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
+ long startTime = System.currentTimeMillis();
CompletableFuture.runAsync(() -> {
for (HivePartition partition : prunedPartitions) {
if (batchException.get() != null || splitAssignment.isStop()) {
@@ -504,6 +534,10 @@ public class HudiScanNode extends HiveScanNode {
splitAssignment.setException(batchException.get());
}
if (numFinishedPartitions.incrementAndGet() ==
prunedPartitions.size()) {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(
+ System.currentTimeMillis() -
startTime);
+ }
splitAssignment.finishSchedule();
}
}
@@ -520,15 +554,10 @@ public class HudiScanNode extends HiveScanNode {
if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) {
return false;
}
- if (!partitionInit) {
- // Non partition table will get one dummy partition
- try {
- prunedPartitions =
hmsTable.getCatalog().getExecutionAuthenticator().execute(()
- -> getPrunedPartitions(hudiClient));
- } catch (Exception e) {
- throw new
RuntimeException(ExceptionUtils.getRootCauseMessage(e), e);
- }
- partitionInit = true;
+ try {
+ initPrunedPartitions();
+ } catch (UserException e) {
+ throw new RuntimeException(e.getMessage(), e);
}
int numPartitions = sessionVariable.getNumPartitionsInBatchMode();
return numPartitions >= 0 && prunedPartitions.size() >= numPartitions;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
index c1626e915c6..65de2f9249d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java
@@ -37,22 +37,20 @@ import java.util.Map;
public class IcebergApiSource implements IcebergSource {
private final ExternalTable targetTable;
- private final Table originTable;
private final TupleDescriptor desc;
+ private Table originTable;
public IcebergApiSource(ExternalTable table, TupleDescriptor desc,
Map<String, ColumnRange> columnNameToRange) {
+ if (!(table instanceof IcebergExternalTable) && !(table instanceof
IcebergSysExternalTable)) {
+ throw new IllegalArgumentException(
+ "Expected Iceberg table but got " +
table.getClass().getSimpleName());
+ }
if (table instanceof IcebergExternalTable) {
IcebergExternalTable icebergExtTable = (IcebergExternalTable)
table;
if (icebergExtTable.isView()) {
throw new UnsupportedOperationException("IcebergApiSource does
not support view");
}
- this.originTable = IcebergUtils.getIcebergTable(icebergExtTable);
- } else if (table instanceof IcebergSysExternalTable) {
- this.originTable = ((IcebergSysExternalTable)
table).getSysIcebergTable();
- } else {
- throw new IllegalArgumentException(
- "Expected Iceberg table but got " +
table.getClass().getSimpleName());
}
this.targetTable = table;
this.desc = desc;
@@ -64,12 +62,19 @@ public class IcebergApiSource implements IcebergSource {
}
@Override
- public String getFileFormat() {
- return IcebergUtils.getFileFormat(originTable).name();
+ public String getFileFormat() throws MetaNotFoundException {
+ return IcebergUtils.getFileFormat(getIcebergTable()).name();
}
@Override
- public Table getIcebergTable() throws MetaNotFoundException {
+ public synchronized Table getIcebergTable() throws MetaNotFoundException {
+ if (originTable == null) {
+ if (targetTable instanceof IcebergExternalTable) {
+ originTable =
IcebergUtils.getIcebergTable((IcebergExternalTable) targetTable);
+ } else {
+ originTable = ((IcebergSysExternalTable)
targetTable).getSysIcebergTable();
+ }
+ }
return originTable;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java
index a87b5408d59..cf33119cd60 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java
@@ -29,12 +29,11 @@ public class IcebergHMSSource implements IcebergSource {
private final HMSExternalTable hmsTable;
private final TupleDescriptor desc;
- private final org.apache.iceberg.Table icebergTable;
+ private org.apache.iceberg.Table icebergTable;
public IcebergHMSSource(HMSExternalTable hmsTable, TupleDescriptor desc) {
this.hmsTable = hmsTable;
this.desc = desc;
- this.icebergTable = IcebergUtils.getIcebergTable(hmsTable);
}
@Override
@@ -44,10 +43,13 @@ public class IcebergHMSSource implements IcebergSource {
@Override
public String getFileFormat() throws DdlException, MetaNotFoundException {
- return IcebergUtils.getFileFormat(icebergTable).name();
+ return IcebergUtils.getFileFormat(getIcebergTable()).name();
}
- public org.apache.iceberg.Table getIcebergTable() throws
MetaNotFoundException {
+ public synchronized org.apache.iceberg.Table getIcebergTable() throws
MetaNotFoundException {
+ if (icebergTable == null) {
+ icebergTable = IcebergUtils.getIcebergTable(hmsTable);
+ }
return icebergTable;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index adc2507e249..9497db0f47f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -213,24 +213,31 @@ public class IcebergScanNode extends FileQueryScanNode {
@Override
protected void doInitialize() throws UserException {
- icebergTable = source.getIcebergTable();
- partitionMapInfos = new HashMap<>();
- isPartitionedTable = icebergTable.spec().isPartitioned();
- // Metadata tables (system tables) are not BaseTable instances, so we
need to handle this case
- if (icebergTable instanceof BaseTable) {
- formatVersion = ((BaseTable)
icebergTable).operations().current().formatVersion();
- } else {
- // For metadata tables (e.g., snapshots, history), use a default
format version
- // These tables are always readable regardless of format version
- formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION;
- }
- preExecutionAuthenticator =
source.getCatalog().getExecutionAuthenticator();
- storagePropertiesMap =
VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials(
-
source.getCatalog().getCatalogProperty().getMetastoreProperties(),
-
source.getCatalog().getCatalogProperty().getStoragePropertiesMap(),
- icebergTable
- );
- backendStorageProperties =
CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap);
+ long startTime = System.currentTimeMillis();
+ try {
+ icebergTable = source.getIcebergTable();
+ partitionMapInfos = new HashMap<>();
+ isPartitionedTable = icebergTable.spec().isPartitioned();
+ // Metadata tables (system tables) are not BaseTable instances, so
we need to handle this case
+ if (icebergTable instanceof BaseTable) {
+ formatVersion = ((BaseTable)
icebergTable).operations().current().formatVersion();
+ } else {
+ // For metadata tables (e.g., snapshots, history), use a
default format version
+ // These tables are always readable regardless of format
version
+ formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION;
+ }
+ preExecutionAuthenticator =
source.getCatalog().getExecutionAuthenticator();
+ storagePropertiesMap =
VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials(
+
source.getCatalog().getCatalogProperty().getMetastoreProperties(),
+
source.getCatalog().getCatalogProperty().getStoragePropertiesMap(),
+ icebergTable
+ );
+ backendStorageProperties =
CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap);
+ } finally {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis()
- startTime);
+ }
+ }
super.doInitialize();
}
@@ -499,15 +506,23 @@ public class IcebergScanNode extends FileQueryScanNode {
try {
preExecutionAuthenticator.execute(
() -> {
- CloseableIterable<FileScanTask> fileScanTasks =
planFileScanTask(scan);
- taskRef.set(fileScanTasks);
-
- CloseableIterator<FileScanTask> iterator =
fileScanTasks.iterator();
- while (splitAssignment.needMoreSplit() &&
iterator.hasNext()) {
- try {
-
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(iterator.next())));
- } catch (UserException e) {
- throw new RuntimeException(e);
+ long startTime = System.currentTimeMillis();
+ try {
+ CloseableIterable<FileScanTask> fileScanTasks
= planFileScanTask(scan);
+ taskRef.set(fileScanTasks);
+ CloseableIterator<FileScanTask> iterator =
fileScanTasks.iterator();
+ while (splitAssignment.needMoreSplit() &&
iterator.hasNext()) {
+ try {
+ splitAssignment.addToQueue(
+
Lists.newArrayList(createIcebergSplit(iterator.next())));
+ } catch (UserException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ } finally {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(
+ System.currentTimeMillis() -
startTime);
}
}
}
@@ -913,10 +928,12 @@ public class IcebergScanNode extends FileQueryScanNode {
// Normal table scan planning
TableScan scan = createTableScan();
+ long startTime = System.currentTimeMillis();
try (CloseableIterable<FileScanTask> fileScanTasks =
planFileScanTask(scan)) {
if (tableLevelPushDownCount) {
int needSplitCnt = countFromSnapshot <
COUNT_WITH_PARALLEL_SPLITS
- ? 1 :
sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()) *
numBackends;
+ ? 1 :
sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName())
+ * numBackends;
for (FileScanTask next : fileScanTasks) {
splits.add(createIcebergSplit(next));
if (splits.size() >= needSplitCnt) {
@@ -932,6 +949,10 @@ public class IcebergScanNode extends FileQueryScanNode {
}
} catch (IOException e) {
throw new UserException(e.getMessage(), e.getCause());
+ } finally {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis()
- startTime);
+ }
}
selectedPartitionNum = partitionMapInfos.size();
@@ -942,10 +963,15 @@ public class IcebergScanNode extends FileQueryScanNode {
private List<Split> doGetSystemTableSplits() throws UserException {
List<Split> splits = new ArrayList<>();
TableScan scan = createTableScan();
+ long startTime = System.currentTimeMillis();
try (CloseableIterable<FileScanTask> fileScanTasks = scan.planFiles())
{
fileScanTasks.forEach(task ->
splits.add(createIcebergSysSplit(task)));
} catch (IOException e) {
throw new UserException(e.getMessage(), e);
+ } finally {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis()
- startTime);
+ }
}
selectedPartitionNum = 0;
return splits;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index bc9f6ead550..3243db01d29 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -161,6 +161,7 @@ public class PaimonScanNode extends FileQueryScanNode {
@Override
protected void doInitialize() throws UserException {
super.doInitialize();
+ long startTime = System.currentTimeMillis();
source = new PaimonSource(desc);
serializedTable =
PaimonUtil.encodeObjectToString(source.getPaimonTable());
// Todo: Get the current schema id of the table, instead of using -1.
@@ -173,6 +174,9 @@ public class PaimonScanNode extends FileQueryScanNode {
);
backendStorageProperties =
CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap);
backendPaimonOptions = getBackendPaimonOptions();
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis()
- startTime);
+ }
}
@VisibleForTesting
@@ -551,30 +555,37 @@ public class PaimonScanNode extends FileQueryScanNode {
@VisibleForTesting
public List<org.apache.paimon.table.source.Split> getPaimonSplitFromAPI()
throws UserException {
- Table paimonTable = getProcessedTable();
- int[] projected = desc.getSlots().stream().mapToInt(
- slot -> paimonTable.rowType()
- .getFieldNames()
- .stream()
- .map(String::toLowerCase)
- .collect(Collectors.toList())
- .indexOf(slot.getColumn().getName()))
- .filter(i -> i >= 0)
- .toArray();
- ReadBuilder readBuilder = paimonTable.newReadBuilder();
- TableScan scan = readBuilder.withFilter(predicates)
- .withProjection(projected)
- .newScan();
- PaimonMetricRegistry registry = new PaimonMetricRegistry();
- if (scan instanceof InnerTableScan) {
- scan = ((InnerTableScan) scan).withMetricRegistry(registry);
- }
- List<org.apache.paimon.table.source.Split> splits =
scan.plan().splits();
- PaimonScanMetricsReporter.report(source.getTargetTable(),
paimonTable.name(), registry);
- if (!registry.getAllGroups().isEmpty()) {
- registry.clear();
+ long startTime = System.currentTimeMillis();
+ try {
+ Table paimonTable = getProcessedTable();
+ int[] projected = desc.getSlots().stream().mapToInt(
+ slot -> paimonTable.rowType()
+ .getFieldNames()
+ .stream()
+ .map(String::toLowerCase)
+ .collect(Collectors.toList())
+ .indexOf(slot.getColumn().getName()))
+ .filter(i -> i >= 0)
+ .toArray();
+ ReadBuilder readBuilder = paimonTable.newReadBuilder();
+ TableScan scan = readBuilder.withFilter(predicates)
+ .withProjection(projected)
+ .newScan();
+ PaimonMetricRegistry registry = new PaimonMetricRegistry();
+ if (scan instanceof InnerTableScan) {
+ scan = ((InnerTableScan) scan).withMetricRegistry(registry);
+ }
+ List<org.apache.paimon.table.source.Split> splits =
scan.plan().splits();
+ PaimonScanMetricsReporter.report(source.getTargetTable(),
paimonTable.name(), registry);
+ if (!registry.getAllGroups().isEmpty()) {
+ registry.clear();
+ }
+ return splits;
+ } finally {
+ if (getSummaryProfile() != null) {
+
getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis()
- startTime);
+ }
}
- return splits;
}
private String getFileFormat(String path) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java
index 24cd69915b5..a809d4d982f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java
@@ -60,4 +60,28 @@ public class SummaryProfileTest {
Assertions.assertEquals(executionSummary.getInfoString(SummaryProfile.SCHEDULE_TIME),
"12ms");
Assertions.assertEquals(executionSummary.getInfoString(SummaryProfile.WAIT_FETCH_RESULT_TIME),
"13ms");
}
+
+ @Test
+ public void testExternalTableMetaSummary() {
+ SummaryProfile profile = new SummaryProfile();
+ profile.addExternalTableGetTableMetaTime(2);
+ profile.addExternalTableGetPartitionValuesTime(3);
+ profile.addExternalTableGetPartitionsTime(5);
+ profile.addExternalTableGetPartitionFilesTime(7);
+ profile.addExternalTableGetFileScanTasksTime(11);
+
+ profile.update(ImmutableMap.of());
+
+ RuntimeProfile executionSummary = profile.getExecutionSummary();
+ Assertions.assertEquals("28ms",
executionSummary.getInfoString(SummaryProfile.EXTERNAL_TABLE_META_TIME));
+ Assertions.assertEquals("2ms", executionSummary.getInfoString(
+ SummaryProfile.EXTERNAL_TABLE_GET_TABLE_META_TIME));
+ Assertions.assertEquals("3ms", executionSummary.getInfoString(
+ SummaryProfile.EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME));
+ Assertions.assertEquals("5ms",
executionSummary.getInfoString(SummaryProfile.GET_PARTITIONS_TIME));
+ Assertions.assertEquals("7ms",
executionSummary.getInfoString(SummaryProfile.GET_PARTITION_FILES_TIME));
+ Assertions.assertEquals("11ms", executionSummary.getInfoString(
+ SummaryProfile.EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME));
+ Assertions.assertEquals(28, profile.getExternalCatalogMetaTimeMs());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
