mcvsubbu commented on a change in pull request #8069:
URL: https://github.com/apache/pinot/pull/8069#discussion_r792128362
##########
File path:
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
##########
@@ -158,47 +162,65 @@ protected synchronized void
deleteSegmentFromPropertyStoreAndLocal(String tableN
if (!segmentsToRetryLater.isEmpty()) {
long effectiveDeletionDelay = Math.min(deletionDelay * 2,
MAX_DELETION_DELAY_SECONDS);
LOGGER.info("Postponing deletion of {} segments from table {}",
segmentsToRetryLater.size(), tableName);
- deleteSegmentsWithDelay(tableName, segmentsToRetryLater,
effectiveDeletionDelay);
+ deleteSegmentsWithDelay(tableName, segmentsToRetryLater,
isInstantDeletion, effectiveDeletionDelay);
return;
}
}
public void removeSegmentsFromStore(String tableNameWithType, List<String>
segments) {
+ removeSegmentsFromStore(tableNameWithType, segments, false);
+ }
+
+ public void removeSegmentsFromStore(String tableNameWithType, List<String>
segments, boolean isInstantDeletion) {
for (String segment : segments) {
- removeSegmentFromStore(tableNameWithType, segment);
+ removeSegmentFromStore(tableNameWithType, segment, isInstantDeletion);
}
}
- protected void removeSegmentFromStore(String tableNameWithType, String
segmentId) {
+ protected void removeSegmentFromStore(String tableNameWithType, String
segmentId, boolean isInstantDeletion) {
// Ignore HLC segments as they are not stored in Pinot FS
if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
return;
}
if (_dataDir != null) {
String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
- URI fileToMoveURI = URIUtils.getUri(_dataDir, rawTableName,
URIUtils.encode(segmentId));
- URI deletedSegmentDestURI = URIUtils.getUri(_dataDir, DELETED_SEGMENTS,
rawTableName, URIUtils.encode(segmentId));
- PinotFS pinotFS = PinotFSFactory.create(fileToMoveURI.getScheme());
-
- try {
- if (pinotFS.exists(fileToMoveURI)) {
- // Overwrites the file if it already exists in the target directory.
- if (pinotFS.move(fileToMoveURI, deletedSegmentDestURI, true)) {
- // Updates last modified.
- // Touch is needed here so that removeAgedDeletedSegments() works
correctly.
- pinotFS.touch(deletedSegmentDestURI);
- LOGGER.info("Moved segment {} from {} to {}", segmentId,
fileToMoveURI.toString(),
- deletedSegmentDestURI.toString());
+ URI fileToDeleteURI = URIUtils.getUri(_dataDir, rawTableName,
URIUtils.encode(segmentId));
+ PinotFS pinotFS = PinotFSFactory.create(fileToDeleteURI.getScheme());
+ if (isInstantDeletion) {
Review comment:
Why not take this from the config for number of days to save the
segment? If the config is 0, then delete it right here. This will avoid
changing the API
##########
File path:
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
##########
@@ -82,22 +82,26 @@ public void stop() {
_executorService.shutdownNow();
}
- public void deleteSegments(final String tableName, final Collection<String>
segmentIds) {
- deleteSegmentsWithDelay(tableName, segmentIds,
DEFAULT_DELETION_DELAY_SECONDS);
+ public void deleteSegments(String tableName, Collection<String> segmentIds) {
+ deleteSegments(tableName, segmentIds, false);
}
- protected void deleteSegmentsWithDelay(final String tableName, final
Collection<String> segmentIds,
- final long deletionDelaySeconds) {
+ public void deleteSegments(String tableName, Collection<String> segmentIds,
boolean isInstantDeletion) {
Review comment:
Suggestion: make the delete segments api with a time argument public,
and call it with 0 (or -1) to indicate "delete right away"
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
