This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4ed96b51de add some metrics for upsert table preloading (#12722)
4ed96b51de is described below
commit 4ed96b51dea7a28b059fca54552b4c7346cd8be7
Author: Xiaobing <61892277+klsi...@users.noreply.github.com>
AuthorDate: Mon Mar 25 16:03:01 2024 -0700
add some metrics for upsert table preloading (#12722)
* add a timer for upsert table preloading
* add failure meter too
---
.../src/main/java/org/apache/pinot/common/metrics/ServerMeter.java | 1 +
.../src/main/java/org/apache/pinot/common/metrics/ServerTimer.java | 2 ++
.../segment/local/upsert/BasePartitionUpsertMetadataManager.java | 5 +++++
3 files changed, 8 insertions(+)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
index e8819ca945..02005a3814 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
@@ -53,6 +53,7 @@ public enum ServerMeter implements AbstractMetrics.Meter {
DELETED_KEYS_TTL_PRIMARY_KEYS_REMOVED("rows", false),
METADATA_TTL_PRIMARY_KEYS_REMOVED("rows", false),
UPSERT_MISSED_VALID_DOC_ID_SNAPSHOT_COUNT("segments", false),
+ UPSERT_PRELOAD_FAILURE("count", false),
ROWS_WITH_ERRORS("rows", false),
LLC_CONTROLLER_RESPONSE_NOT_SENT("messages", true),
LLC_CONTROLLER_RESPONSE_COMMIT("messages", true),
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
index 6093a0f76b..8dbfd706bd 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerTimer.java
@@ -49,6 +49,8 @@ public enum ServerTimer implements AbstractMetrics.Timer {
TOTAL_CPU_TIME_NS("nanoseconds", false, "Total query cost (thread cpu time +
system "
+ "activities cpu time + response serialization cpu time) for query
processing on server."),
+ UPSERT_PRELOAD_TIME_MS("milliseconds", false,
+ "Total time taken to preload a table partition of an upsert table with
upsert snapshot"),
UPSERT_REMOVE_EXPIRED_PRIMARY_KEYS_TIME_MS("milliseconds", false,
"Total time taken to delete expired primary keys based on metadataTTL or
deletedKeysTTL"),
UPSERT_SNAPSHOT_TIME_MS("milliseconds", false, "Total time taken to take
upsert table snapshot");
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
index a97bb65922..81e5dbaecd 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
@@ -199,7 +199,11 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
return;
}
// From now on, the _isPreloading flag is true until the segments are
preloaded.
+ long startTime = System.currentTimeMillis();
doPreloadSegments(tableDataManager, indexLoadingConfig, helixManager,
segmentPreloadExecutor);
+ long duration = System.currentTimeMillis() - startTime;
+ _serverMetrics.addTimedTableValue(_tableNameWithType,
ServerTimer.UPSERT_PRELOAD_TIME_MS, duration,
+ TimeUnit.MILLISECONDS);
} catch (Exception e) {
// Even if preloading fails, we should continue to complete the
initialization, so that TableDataManager can be
// created. Once TableDataManager is created, no more segment preloading
would happen, and the normal segment
@@ -207,6 +211,7 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
// normal segment loading logic, the one doing more costly checks on the
upsert metadata.
_logger.warn("Failed to preload segments from partition: {} of table:
{}, skipping", _partitionId,
_tableNameWithType, e);
+ _serverMetrics.addMeteredTableValue(_tableNameWithType,
ServerMeter.UPSERT_PRELOAD_FAILURE, 1);
if (e instanceof InterruptedException) {
// Restore the interrupted status in case the upper callers want to
check.
Thread.currentThread().interrupt();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
