This is an automated email from the ASF dual-hosted git repository.
somandal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cce8c8d3e83 Add metric to track latency for segment download (#16375)
cce8c8d3e83 is described below
commit cce8c8d3e8372d5755dcbea0f2a5a85ff555f68f
Author: Songqiao Su <[email protected]>
AuthorDate: Thu Jul 17 12:54:06 2025 -0700
Add metric to track latency for segment download (#16375)
---
.../main/java/org/apache/pinot/common/metrics/ServerTimer.java | 7 ++++++-
.../org/apache/pinot/core/data/manager/BaseTableDataManager.java | 9 +++++++++
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
index 917a1879468..0f72acd8264 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
@@ -101,7 +101,12 @@ public enum ServerTimer implements AbstractMetrics.Timer {
STARTUP_SUCCESS_DURATION_MS("millis", true),
STARTUP_FAILURE_DURATION_MS("millis", true),
- PREDOWNLOAD_TIME("millis", true);
+ PREDOWNLOAD_TIME("millis", true),
+
+ SEGMENT_DOWNLOAD_FROM_DEEP_STORE_TIME_MS("millis", false,
+ "Time taken to download a segment from deep store (including untar and
move operations)"),
+ SEGMENT_DOWNLOAD_FROM_PEERS_TIME_MS("millis", false,
+ "Time taken to download a segment from peers (including untar and move
operations)");
private final String _timerName;
private final boolean _global;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index b5495b57232..c809da80ddf 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -57,6 +57,7 @@ import
org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.metrics.ServerGauge;
import org.apache.pinot.common.metrics.ServerMeter;
import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.metrics.ServerTimer;
import org.apache.pinot.common.restlet.resources.SegmentErrorInfo;
import org.apache.pinot.common.utils.TarCompressionUtils;
import org.apache.pinot.common.utils.config.TierConfigUtils;
@@ -961,6 +962,7 @@ public abstract class BaseTableDataManager implements
TableDataManager {
}
_serverMetrics.setValueOfTableGauge(_tableNameWithType,
ServerGauge.SEGMENT_TABLE_DOWNLOAD_COUNT,
_numSegmentsAcquiredDownloadSemaphore.incrementAndGet());
+ long downloadStartTime = System.currentTimeMillis();
try {
File untarredSegmentDir;
if (_isStreamSegmentDownloadUntar && zkMetadata.getCrypterName() ==
null) {
@@ -993,6 +995,9 @@ public abstract class BaseTableDataManager implements
TableDataManager {
if (_segmentOperationsThrottler != null) {
_segmentOperationsThrottler.getSegmentDownloadThrottler().release();
}
+ long downloadDuration = System.currentTimeMillis() - downloadStartTime;
+ _serverMetrics.addTimedTableValue(_tableNameWithType,
ServerTimer.SEGMENT_DOWNLOAD_FROM_DEEP_STORE_TIME_MS,
+ downloadDuration, TimeUnit.MILLISECONDS);
FileUtils.deleteQuietly(tempRootDir);
}
} finally {
@@ -1022,6 +1027,7 @@ public abstract class BaseTableDataManager implements
TableDataManager {
+ "(lock-time={}ms, queue-length={}).", segmentName,
System.currentTimeMillis() - startTime,
segmentDownloadThrottler.getQueueLength());
}
+ long downloadStartTime = System.currentTimeMillis();
try {
SegmentFetcherFactory.fetchAndDecryptSegmentToLocal(segmentName,
_peerDownloadScheme, () -> {
List<URI> peerServerURIs =
@@ -1042,6 +1048,9 @@ public abstract class BaseTableDataManager implements
TableDataManager {
if (_segmentOperationsThrottler != null) {
_segmentOperationsThrottler.getSegmentDownloadThrottler().release();
}
+ long downloadDuration = System.currentTimeMillis() - downloadStartTime;
+ _serverMetrics.addTimedTableValue(_tableNameWithType,
ServerTimer.SEGMENT_DOWNLOAD_FROM_PEERS_TIME_MS,
+ downloadDuration, TimeUnit.MILLISECONDS);
FileUtils.deleteQuietly(tempRootDir);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
