This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 444e445dbf Add metric to track the number of primary keys in valid doc
id snapshot (#11515)
444e445dbf is described below
commit 444e445dbf4df9173e11f1be2d9a44ad53d8dc23
Author: Kartik Khare <[email protected]>
AuthorDate: Wed Sep 6 20:40:39 2023 +0530
Add metric to track the number of primary keys in valid doc id snapshot
(#11515)
* Add metric to track the number of primary keys in valid doc id snapshot
* Use long instead of int for metric
---------
Co-authored-by: Kartik Khare <[email protected]>
---
.../src/main/java/org/apache/pinot/common/metrics/ServerGauge.java | 3 ++-
.../segment/local/upsert/BasePartitionUpsertMetadataManager.java | 5 +++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
index cf8a2a304d..c05baa0cca 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
@@ -50,7 +50,8 @@ public enum ServerGauge implements AbstractMetrics.Gauge {
REALTIME_INGESTION_DELAY_MS("milliseconds", false),
END_TO_END_REALTIME_INGESTION_DELAY_MS("milliseconds", false),
// Needed to track if valid doc id snapshots are present for faster restarts
- UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT("upsertValidDocIdSnapshotCount", false);
+ UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT("upsertValidDocIdSnapshotCount", false),
+ UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT("upsertPrimaryKeysInSnapshotCount",
false);
private final String _gaugeName;
private final String _unit;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
index 2b7b36e9ef..e968421537 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
@@ -588,6 +588,7 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
// TODO: Consider optimizing it by tracking and persisting only the changed
snapshot
protected void doTakeSnapshot() {
int numTrackedSegments = _trackedSegments.size();
+ long numPrimaryKeysInSnapshot = 0L;
_logger.info("Taking snapshot for {} segments", numTrackedSegments);
long startTimeMs = System.currentTimeMillis();
@@ -596,11 +597,15 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
if (segment instanceof ImmutableSegmentImpl) {
((ImmutableSegmentImpl) segment).persistValidDocIdsSnapshot();
numImmutableSegments++;
+ numPrimaryKeysInSnapshot +=
+ ((ImmutableSegmentImpl)
segment).getValidDocIds().getMutableRoaringBitmap().getCardinality();
}
}
_serverMetrics.setValueOfPartitionGauge(_tableNameWithType, _partitionId,
ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, numImmutableSegments);
+ _serverMetrics.setValueOfPartitionGauge(_tableNameWithType, _partitionId,
+ ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT,
numPrimaryKeysInSnapshot);
_logger.info("Finished taking snapshot for {} immutable segments (out of
{} total segments) in {}ms",
numImmutableSegments, numTrackedSegments, System.currentTimeMillis() -
startTimeMs);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
