mcvsubbu commented on a change in pull request #8069:
URL: https://github.com/apache/pinot/pull/8069#discussion_r792205799
##########
File path:
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
##########
@@ -158,47 +162,65 @@ protected synchronized void
deleteSegmentFromPropertyStoreAndLocal(String tableN
if (!segmentsToRetryLater.isEmpty()) {
long effectiveDeletionDelay = Math.min(deletionDelay * 2,
MAX_DELETION_DELAY_SECONDS);
LOGGER.info("Postponing deletion of {} segments from table {}",
segmentsToRetryLater.size(), tableName);
- deleteSegmentsWithDelay(tableName, segmentsToRetryLater,
effectiveDeletionDelay);
+ deleteSegmentsWithDelay(tableName, segmentsToRetryLater,
isInstantDeletion, effectiveDeletionDelay);
return;
}
}
public void removeSegmentsFromStore(String tableNameWithType, List<String>
segments) {
+ removeSegmentsFromStore(tableNameWithType, segments, false);
+ }
+
+ public void removeSegmentsFromStore(String tableNameWithType, List<String>
segments, boolean isInstantDeletion) {
for (String segment : segments) {
- removeSegmentFromStore(tableNameWithType, segment);
+ removeSegmentFromStore(tableNameWithType, segment, isInstantDeletion);
}
}
- protected void removeSegmentFromStore(String tableNameWithType, String
segmentId) {
+ protected void removeSegmentFromStore(String tableNameWithType, String
segmentId, boolean isInstantDeletion) {
// Ignore HLC segments as they are not stored in Pinot FS
if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
return;
}
if (_dataDir != null) {
String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
- URI fileToMoveURI = URIUtils.getUri(_dataDir, rawTableName,
URIUtils.encode(segmentId));
- URI deletedSegmentDestURI = URIUtils.getUri(_dataDir, DELETED_SEGMENTS,
rawTableName, URIUtils.encode(segmentId));
- PinotFS pinotFS = PinotFSFactory.create(fileToMoveURI.getScheme());
-
- try {
- if (pinotFS.exists(fileToMoveURI)) {
- // Overwrites the file if it already exists in the target directory.
- if (pinotFS.move(fileToMoveURI, deletedSegmentDestURI, true)) {
- // Updates last modified.
- // Touch is needed here so that removeAgedDeletedSegments() works
correctly.
- pinotFS.touch(deletedSegmentDestURI);
- LOGGER.info("Moved segment {} from {} to {}", segmentId,
fileToMoveURI.toString(),
- deletedSegmentDestURI.toString());
+ URI fileToDeleteURI = URIUtils.getUri(_dataDir, rawTableName,
URIUtils.encode(segmentId));
+ PinotFS pinotFS = PinotFSFactory.create(fileToDeleteURI.getScheme());
+ if (isInstantDeletion) {
Review comment:
Then maybe add a table config to override the setting, so that the
behavior is the same for all segments in the table, and not dependent on how
someone chooses to delete a segment?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
