This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 16e97c5668 untrack the segment out of TTL (#12449)
16e97c5668 is described below
commit 16e97c5668379d304fa052ca8183d9f4f2a402e3
Author: Xiaobing <[email protected]>
AuthorDate: Tue Feb 20 19:06:01 2024 -0800
untrack the segment out of TTL (#12449)
---
.../local/upsert/BasePartitionUpsertMetadataManager.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
index cdf2a61fe9..47dd64f49d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
@@ -513,13 +513,16 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
_logger.info("Skip removing untracked (replaced or empty) segment: {}",
segmentName);
return;
}
- // Skip removing segment that has max comparison value smaller than
(largestSeenComparisonValue - TTL)
+ // Skip removing the upsert metadata of segment that has max comparison
value smaller than
+ // (largestSeenComparisonValue - TTL), i.e. out of metadata TTL. The
expired metadata is removed while creating
+ // new consuming segment in batches.
+ boolean skipRemoveMetadata = false;
if (_metadataTTL > 0 && _largestSeenComparisonValue.get() > 0) {
Number maxComparisonValue =
(Number)
segment.getSegmentMetadata().getColumnMetadataMap().get(_comparisonColumns.get(0)).getMaxValue();
if (maxComparisonValue.doubleValue() < _largestSeenComparisonValue.get()
- _metadataTTL) {
_logger.info("Skip removing segment: {} because it's out of TTL",
segmentName);
- return;
+ skipRemoveMetadata = true;
}
}
if (!startOperation()) {
@@ -530,7 +533,9 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
_snapshotLock.readLock().lock();
}
try {
- doRemoveSegment(segment);
+ if (!skipRemoveMetadata) {
+ doRemoveSegment(segment);
+ }
_trackedSegments.remove(segment);
} finally {
if (_enableSnapshot) {
@@ -708,8 +713,7 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, numImmutableSegments);
_serverMetrics.setValueOfPartitionGauge(_tableNameWithType, _partitionId,
ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT,
numPrimaryKeysInSnapshot);
- _logger.info(
- "Finished taking snapshot for {} immutable segments with {} primary
keys (out of {} total segments, "
+ _logger.info("Finished taking snapshot for {} immutable segments with {}
primary keys (out of {} total segments, "
+ "{} are consuming segments) in {} ms", numImmutableSegments,
numPrimaryKeysInSnapshot, numTrackedSegments,
numConsumingSegments, System.currentTimeMillis() - startTimeMs);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
