swamirishi commented on code in PR #9423:
URL: https://github.com/apache/ozone/pull/9423#discussion_r2613354692
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java:
##########
@@ -182,6 +187,15 @@ public void processPaths(
deletedKey, repeatedOmKeyInfo);
}
+ for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubFilesList()) {
+ keySpaceOmMetadataManager.getKeyTable(getBucketLayout())
+ .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(),
keyRanges.getValue());
+ if (LOG.isDebugEnabled()) {
Review Comment:
same
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java:
##########
@@ -27,12 +30,35 @@
public class DeleteKeysResult {
private List<OmKeyInfo> keysToDelete;
-
private boolean processedKeys;
+ private List<ExclusiveRange> keyRanges;
+ private static final Logger LOG =
LoggerFactory.getLogger(DeleteKeysResult.class);
- public DeleteKeysResult(List<OmKeyInfo> keysToDelete, boolean processedKeys)
{
+ public DeleteKeysResult(List<OmKeyInfo> keysToDelete, List<ExclusiveRange>
keyRanges, boolean processedKeys) {
this.keysToDelete = keysToDelete;
this.processedKeys = processedKeys;
+ this.keyRanges = keyRanges;
+ validateNonOverlappingRanges();
+ }
+
+ private void validateNonOverlappingRanges() {
+ if (keyRanges == null || keyRanges.size() <= 1) {
+ return;
+ }
+ String lastEnd = null;
+ for (ExclusiveRange range : keyRanges) {
+ if (range == null || range.getStartKey() == null ||
range.getExclusiveEndKey() == null) {
Review Comment:
why would ranges ever be null? We should not continue. We should throw an
exception if range is null or startRange is null or endKey is null. This might
end up cleaning up the entire DB
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java:
##########
@@ -27,12 +30,35 @@
public class DeleteKeysResult {
private List<OmKeyInfo> keysToDelete;
-
private boolean processedKeys;
+ private List<ExclusiveRange> keyRanges;
+ private static final Logger LOG =
LoggerFactory.getLogger(DeleteKeysResult.class);
- public DeleteKeysResult(List<OmKeyInfo> keysToDelete, boolean processedKeys)
{
+ public DeleteKeysResult(List<OmKeyInfo> keysToDelete, List<ExclusiveRange>
keyRanges, boolean processedKeys) {
this.keysToDelete = keysToDelete;
this.processedKeys = processedKeys;
+ this.keyRanges = keyRanges;
+ validateNonOverlappingRanges();
+ }
+
+ private void validateNonOverlappingRanges() {
+ if (keyRanges == null || keyRanges.size() <= 1) {
+ return;
+ }
+ String lastEnd = null;
+ for (ExclusiveRange range : keyRanges) {
+ if (range == null || range.getStartKey() == null ||
range.getExclusiveEndKey() == null) {
+ continue;
+ }
+ if (lastEnd != null && range.getStartKey().compareTo(lastEnd) < 0) {
+ LOG.warn(
+ "Overlapping or unsorted delete ranges detected. " + "Clearing
keyRanges to avoid incorrect deleteRange. " +
+ "previousEnd={}, currentStart={}", lastEnd,
range.getStartKey());
+ keyRanges = Collections.emptyList();
Review Comment:
What about KeysToDelete? This will create orphan subFiles and subDirectories.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java:
##########
@@ -27,12 +30,35 @@
public class DeleteKeysResult {
private List<OmKeyInfo> keysToDelete;
-
private boolean processedKeys;
+ private List<ExclusiveRange> keyRanges;
+ private static final Logger LOG =
LoggerFactory.getLogger(DeleteKeysResult.class);
- public DeleteKeysResult(List<OmKeyInfo> keysToDelete, boolean processedKeys)
{
+ public DeleteKeysResult(List<OmKeyInfo> keysToDelete, List<ExclusiveRange>
keyRanges, boolean processedKeys) {
this.keysToDelete = keysToDelete;
this.processedKeys = processedKeys;
+ this.keyRanges = keyRanges;
+ validateNonOverlappingRanges();
+ }
+
+ private void validateNonOverlappingRanges() {
Review Comment:
Do we actually need this validation? Things need not be always sorted it
depends on the rocksdb bytewise comparator order. Rocksdb would guarantee us
that already
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java:
##########
@@ -147,25 +148,29 @@ public void processPaths(
deletedSpaceOmMetadataManager.getDeletedDirTable().putWithBatch(deletedSpaceBatchOperation,
ozoneDeleteKey, keyInfo);
-
keySpaceOmMetadataManager.getDirectoryTable().deleteWithBatch(keySpaceBatchOperation,
- ozoneDbKey);
-
if (LOG.isDebugEnabled()) {
LOG.debug("markDeletedDirList KeyName: {}, DBKey: {}",
keyInfo.getKeyName(), ozoneDbKey);
}
}
+ for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubDirsList()) {
+ keySpaceOmMetadataManager.getDirectoryTable()
+ .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(),
keyRanges.getValue());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sub Directory delete range Start Key(inclusive): {} and
End Key(exclusive): {}",
+ keyRanges.getKey(), keyRanges.getValue());
+ }
+ }
+
for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) {
OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key)
.withCommittedKeyDeletedFlag(true);
String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId,
bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName());
- keySpaceOmMetadataManager.getKeyTable(getBucketLayout())
- .deleteWithBatch(keySpaceBatchOperation, ozoneDbKey);
if (LOG.isDebugEnabled()) {
- LOG.info("Move keyName:{} to DeletedTable DBKey: {}",
+ LOG.debug("Move keyName:{} to DeletedTable DBKey: {}",
Review Comment:
nit: the if check is not required lets remove it.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java:
##########
@@ -27,12 +30,35 @@
public class DeleteKeysResult {
private List<OmKeyInfo> keysToDelete;
-
private boolean processedKeys;
+ private List<ExclusiveRange> keyRanges;
+ private static final Logger LOG =
LoggerFactory.getLogger(DeleteKeysResult.class);
- public DeleteKeysResult(List<OmKeyInfo> keysToDelete, boolean processedKeys)
{
+ public DeleteKeysResult(List<OmKeyInfo> keysToDelete, List<ExclusiveRange>
keyRanges, boolean processedKeys) {
this.keysToDelete = keysToDelete;
this.processedKeys = processedKeys;
+ this.keyRanges = keyRanges;
+ validateNonOverlappingRanges();
+ }
+
+ private void validateNonOverlappingRanges() {
Review Comment:
nit: Pass keyRanges in the argument this might throw NPE if someone moves
this method in the constructor later on
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
