hemantk-12 commented on code in PR #5454: URL: https://github.com/apache/ozone/pull/5454#discussion_r1362516492
########## hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/ThrowableFunction.java: ########## @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.utils; + +/** + * Utility interface for function which throws exceptions. + * Similar to java.util.Function. Review Comment: Correct import is ` java.util.function.Function` and it would be better if you use `{@link java.util.function.Function}`. Also annotate is with `@FunctionalInterface` similar to https://stackoverflow.com/a/18198349 ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java: ########## @@ -535,42 +536,37 @@ public SnapshotDiffCleanupService getSnapshotDiffCleanupService() { * @param keyPrefix DB key prefix String * @return endKey String, or null if no keys with such prefix is found */ - private static String findEndKeyGivenPrefix( + private static String performOperationGivenPrefix( TableIterator<String, ? extends Table.KeyValue<String, ?>> keyIter, - String keyPrefix) throws IOException { - - String endKey; + String keyPrefix, ThrowableFunction<Table.KeyValue<String, ?>, + Void, IOException> operationFunction) throws IOException { + String endKey = null; Review Comment: `endKey` is not used anywhere. Also return type could be void now. ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java: ########## @@ -535,42 +536,37 @@ public SnapshotDiffCleanupService getSnapshotDiffCleanupService() { * @param keyPrefix DB key prefix String * @return endKey String, or null if no keys with such prefix is found */ - private static String findEndKeyGivenPrefix( + private static String performOperationGivenPrefix( TableIterator<String, ? extends Table.KeyValue<String, ?>> keyIter, - String keyPrefix) throws IOException { - - String endKey; + String keyPrefix, ThrowableFunction<Table.KeyValue<String, ?>, + Void, IOException> operationFunction) throws IOException { + String endKey = null; keyIter.seek(keyPrefix); Review Comment: I think you can just use[TableIterator#seekToFirst()](https://github.com/apache/ozone/blob/4b9b10f6206ed5c29c0c4957a651de1d24fe86f6/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStoreAbstractIterator.java#L117), if you wanna get all keys starts with prefix. ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java: ########## @@ -473,24 +474,24 @@ public static DBCheckpoint createOmSnapshotCheckpoint( * @param bucketName bucket name */ private static void deleteKeysFromDelDirTableInSnapshotScope( - OMMetadataManager omMetadataManager, - String volumeName, + OMMetadataManager omMetadataManager, String volumeName, String bucketName) throws IOException { // Range delete start key (inclusive) final String beginKey = getOzonePathKeyWithVolumeBucketNames( omMetadataManager, volumeName, bucketName); - // Range delete end key (exclusive). To be calculated - String endKey; try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> - iter = omMetadataManager.getDeletedDirTable().iterator()) { - endKey = findEndKeyGivenPrefix(iter, beginKey); + iter = omMetadataManager.getDeletedDirTable().iterator(beginKey)) { Review Comment: `Table#iterator()` takes `keyPrefix` not `beginKey`. `beginKey` is used as key prefix here and begin key at line number 544. While in actual `beginKey` is just a key prefix. I think we should just call is `keyPrefix`. ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java: ########## @@ -473,24 +474,24 @@ public static DBCheckpoint createOmSnapshotCheckpoint( * @param bucketName bucket name */ private static void deleteKeysFromDelDirTableInSnapshotScope( - OMMetadataManager omMetadataManager, - String volumeName, + OMMetadataManager omMetadataManager, String volumeName, String bucketName) throws IOException { // Range delete start key (inclusive) final String beginKey = getOzonePathKeyWithVolumeBucketNames( omMetadataManager, volumeName, bucketName); - // Range delete end key (exclusive). To be calculated - String endKey; try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> - iter = omMetadataManager.getDeletedDirTable().iterator()) { - endKey = findEndKeyGivenPrefix(iter, beginKey); + iter = omMetadataManager.getDeletedDirTable().iterator(beginKey)) { + performOperationGivenPrefix(iter, beginKey, + entry -> { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing key {} from DeletedDirTable", entry.getKey()); + } + omMetadataManager.getDeletedDirTable().delete(entry.getKey()); + return null; + }); } - - // Clean up deletedDirectoryTable - deleteRangeInclusive(omMetadataManager.getDeletedDirTable(), Review Comment: Delete this method. It is not used anymore. ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java: ########## @@ -535,42 +536,37 @@ public SnapshotDiffCleanupService getSnapshotDiffCleanupService() { * @param keyPrefix DB key prefix String * @return endKey String, or null if no keys with such prefix is found */ - private static String findEndKeyGivenPrefix( + private static String performOperationGivenPrefix( TableIterator<String, ? extends Table.KeyValue<String, ?>> keyIter, - String keyPrefix) throws IOException { - - String endKey; + String keyPrefix, ThrowableFunction<Table.KeyValue<String, ?>, + Void, IOException> operationFunction) throws IOException { + String endKey = null; keyIter.seek(keyPrefix); // Continue only when there are entries of snapshot (bucket) scope // in deletedTable in the first place - if (!keyIter.hasNext()) { - // No key matching keyPrefix. No need to do delete or deleteRange at all. - endKey = null; - } else { - // Remember the last key with a matching prefix - endKey = keyIter.next().getKey(); - - // Loop until prefix mismatches. - // TODO: [SNAPSHOT] Try to seek to next predicted bucket name instead of - // the while-loop for a potential speed up? - // Start performance tracking timer - long startTime = System.nanoTime(); - while (keyIter.hasNext()) { - Table.KeyValue<String, ?> entry = keyIter.next(); - String dbKey = entry.getKey(); - if (dbKey.startsWith(keyPrefix)) { - endKey = dbKey; - } - } - // Time took for the iterator to finish (in ns) - long timeElapsed = System.nanoTime() - startTime; - if (timeElapsed >= DB_TABLE_ITER_LOOP_THRESHOLD_NS) { - // Print time elapsed - LOG.warn("Took {} ns to find endKey. Caller is {}", timeElapsed, - new Throwable().fillInStackTrace().getStackTrace()[1] - .getMethodName()); + // Loop until prefix matches. + // TODO: [SNAPSHOT] Try to seek to next predicted bucket name instead of + // the while-loop for a potential speed up? + // Start performance tracking timer + long startTime = System.nanoTime(); + while (keyIter.hasNext()) { + Table.KeyValue<String, ?> entry = keyIter.next(); + String dbKey = entry.getKey(); + if (dbKey.startsWith(keyPrefix)) { + operationFunction.apply(entry); + endKey = dbKey; + } else { + break; Review Comment: Do we need to do this check here again? It is done in [TableIterator#hasNext()](https://github.com/apache/ozone/blob/4b9b10f6206ed5c29c0c4957a651de1d24fe86f6/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStoreAbstractIterator.java#L97) if prefix is passed in its creation. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
